hbkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hbkit/__init__.py +7 -0
- hbkit/archive.py +387 -0
- hbkit/cli.py +206 -0
- hbkit/doctor.py +208 -0
- hbkit/index.py +203 -0
- hbkit/runner.py +192 -0
- hbkit/tui.py +738 -0
- hbkit-0.1.0.dist-info/METADATA +206 -0
- hbkit-0.1.0.dist-info/RECORD +12 -0
- hbkit-0.1.0.dist-info/WHEEL +4 -0
- hbkit-0.1.0.dist-info/entry_points.txt +3 -0
- hbkit-0.1.0.dist-info/licenses/LICENSE +21 -0
hbkit/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""hbkit - recover files from Synology Hyper Backup (.hbk) archives without Synology software."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.1.0"
|
|
4
|
+
|
|
5
|
+
from .archive import Archive, UnsupportedArchive, find_archive_root, is_archive # noqa: F401
|
|
6
|
+
|
|
7
|
+
__all__ = ["Archive", "UnsupportedArchive", "find_archive_root", "is_archive", "__version__"]
|
hbkit/archive.py
ADDED
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
"""Read a Synology Hyper Backup (.hbk) archive directly. No Synology software required.
|
|
2
|
+
|
|
3
|
+
Format (reverse engineered; all integers BIG-ENDIAN):
|
|
4
|
+
version_list.off_virtual_file -> virtual_file.index record (56B)
|
|
5
|
+
i64 @0 = (file_chunk_index_id << 48) | byte_offset
|
|
6
|
+
-> Config/file_chunk<id>.index @byte_offset : flat array of BE i64 keys, 8B each
|
|
7
|
+
-> each key = byte offset into Pool/chunk_index (29B records)
|
|
8
|
+
u8 mode @0 ; if mode&1 -> i64 cite_offset @1 (indirect, recurse)
|
|
9
|
+
else -> i32 bucket_id @1, i32 bucket_index_offset @5
|
|
10
|
+
-> Pool/<pool>/<bucket_id>>11>/<bucket_id & 0x7FF>.index @bucket_index_offset (32B record)
|
|
11
|
+
u32 comp_len, u32 offset, u32 uncomp_len, md5[16] of DECOMPRESSED chunk,
|
|
12
|
+
u32 crc32 of record[0:28]
|
|
13
|
+
-> same-named .bucket @offset, comp_len bytes: raw LZ4 BLOCK (zlib is the fallback codec;
|
|
14
|
+
SYNO::Backup::decompress dispatches type 1/2/4 = lz4 / lz4-hc / zlib)
|
|
15
|
+
|
|
16
|
+
Index families are <N>.idx shards concatenated in numeric order; records start at stream
|
|
17
|
+
offset 64; only shard 0 carries the 64-byte header (magic 0x7053A86E).
|
|
18
|
+
|
|
19
|
+
Synology writes a generation suffix on most files (`0.idx.2`, `1.db.2`, `index_ver.json.1`).
|
|
20
|
+
Everything here resolves `name`, `name.1`, `name.2`... transparently, preferring the highest
|
|
21
|
+
generation, and ignores macOS AppleDouble junk (`._*`).
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import ctypes
|
|
26
|
+
import hashlib
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
import struct
|
|
30
|
+
import zlib
|
|
31
|
+
from collections import OrderedDict
|
|
32
|
+
|
|
33
|
+
class UnsupportedArchive(Exception):
|
|
34
|
+
"""The archive uses a layout this reader does not implement. Never guess - a wrong
|
|
35
|
+
guess here means silently wrong bytes, which is the one outcome a recovery tool
|
|
36
|
+
must never produce."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
MAX_OPEN_BUCKETS = int(os.environ.get("HBK_MAX_OPEN", "64"))
|
|
40
|
+
BUCKET_BUF = int(os.environ.get("HBK_BUCKET_BUF", 1 << 22)) # read-ahead; platter drives care
|
|
41
|
+
INDEX_BUF = int(os.environ.get("HBK_INDEX_BUF", 1 << 16)) # chunk keys are mostly sequential;
|
|
42
|
+
# unbuffered = one syscall per 29 B
|
|
43
|
+
MAGIC = 0x7053A86E
|
|
44
|
+
|
|
45
|
+
_LZ4_CANDIDATES = [
|
|
46
|
+
os.environ.get("HBK_LZ4"),
|
|
47
|
+
"/opt/homebrew/lib/liblz4.dylib",
|
|
48
|
+
"/usr/local/lib/liblz4.dylib",
|
|
49
|
+
"/usr/lib/x86_64-linux-gnu/liblz4.so.1",
|
|
50
|
+
"liblz4.so.1", "liblz4.dylib",
|
|
51
|
+
]
|
|
52
|
+
_lz4 = None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def lz4():
|
|
56
|
+
"""Lazily bind liblz4. Note the x86_64 dylib inside HyperBackupExplorer.app will
|
|
57
|
+
not load on Apple Silicon - use a native one (brew install lz4)."""
|
|
58
|
+
global _lz4
|
|
59
|
+
if _lz4 is None:
|
|
60
|
+
for c in _LZ4_CANDIDATES:
|
|
61
|
+
if not c:
|
|
62
|
+
continue
|
|
63
|
+
try:
|
|
64
|
+
lib = ctypes.CDLL(c)
|
|
65
|
+
except OSError:
|
|
66
|
+
continue
|
|
67
|
+
lib.LZ4_decompress_safe.argtypes = [ctypes.c_char_p, ctypes.c_char_p,
|
|
68
|
+
ctypes.c_int, ctypes.c_int]
|
|
69
|
+
lib.LZ4_decompress_safe.restype = ctypes.c_int
|
|
70
|
+
_lz4 = lib
|
|
71
|
+
break
|
|
72
|
+
else:
|
|
73
|
+
raise RuntimeError("liblz4 not found - set HBK_LZ4 to its path (brew install lz4)")
|
|
74
|
+
return _lz4
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# ------------------------------------------------------------ path resolution
|
|
78
|
+
|
|
79
|
+
_SHARD = re.compile(r"^(\d+)\.idx(?:\.(\d+))?$")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def resolve(d: str, base: str) -> str:
|
|
83
|
+
"""Find `base` / `base.<gen>` inside directory `d`, preferring the highest generation."""
|
|
84
|
+
best = None
|
|
85
|
+
try:
|
|
86
|
+
names = os.listdir(d)
|
|
87
|
+
except OSError as e:
|
|
88
|
+
raise FileNotFoundError(f"{d}: {e}") from e
|
|
89
|
+
for f in names:
|
|
90
|
+
if f.startswith("._"):
|
|
91
|
+
continue
|
|
92
|
+
if f == base:
|
|
93
|
+
gen = 0
|
|
94
|
+
elif f.startswith(base + "."):
|
|
95
|
+
suf = f[len(base) + 1:]
|
|
96
|
+
if not suf.isdigit():
|
|
97
|
+
continue
|
|
98
|
+
gen = int(suf)
|
|
99
|
+
else:
|
|
100
|
+
continue
|
|
101
|
+
if best is None or gen > best[0]:
|
|
102
|
+
best = (gen, f)
|
|
103
|
+
if best is None:
|
|
104
|
+
raise FileNotFoundError(f"no {base}[.gen] in {d}")
|
|
105
|
+
return os.path.join(d, best[1])
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class Cat:
|
|
109
|
+
"""Numbered <N>.idx[.gen] shards presented as a single logical byte stream."""
|
|
110
|
+
|
|
111
|
+
def __init__(self, d: str):
|
|
112
|
+
self.d = d
|
|
113
|
+
found: dict[int, tuple[int, str]] = {}
|
|
114
|
+
for f in os.listdir(d):
|
|
115
|
+
if f.startswith("._"):
|
|
116
|
+
continue
|
|
117
|
+
m = _SHARD.match(f)
|
|
118
|
+
if not m:
|
|
119
|
+
continue
|
|
120
|
+
n, gen = int(m.group(1)), int(m.group(2) or 0)
|
|
121
|
+
if n not in found or gen > found[n][0]:
|
|
122
|
+
found[n] = (gen, f)
|
|
123
|
+
if not found:
|
|
124
|
+
raise FileNotFoundError(f"no .idx shards in {d}")
|
|
125
|
+
self.files = [found[k][1] for k in sorted(found)]
|
|
126
|
+
self.sizes = [os.path.getsize(os.path.join(d, f)) for f in self.files]
|
|
127
|
+
self.starts, t = [], 0
|
|
128
|
+
for s in self.sizes:
|
|
129
|
+
self.starts.append(t)
|
|
130
|
+
t += s
|
|
131
|
+
self.total = t
|
|
132
|
+
self._fh: dict[int, object] = {}
|
|
133
|
+
# Shard 0 carries a 64-byte header that declares the record size, so readers do
|
|
134
|
+
# not have to hardcode per-DSM-version layouts.
|
|
135
|
+
with open(os.path.join(d, self.files[0]), "rb") as fh:
|
|
136
|
+
head = fh.read(64)
|
|
137
|
+
if len(head) < 64:
|
|
138
|
+
raise UnsupportedArchive(f"{d}: shard 0 is shorter than its header")
|
|
139
|
+
w = struct.unpack(">16I", head)
|
|
140
|
+
if w[0] != MAGIC:
|
|
141
|
+
raise UnsupportedArchive(f"{d}: bad magic {w[0]:#010x}, expected {MAGIC:#010x}")
|
|
142
|
+
self.kind, self.variant = w[1], w[2]
|
|
143
|
+
self.record_size = w[4]
|
|
144
|
+
self.declared_total = (w[5] << 32) | w[6]
|
|
145
|
+
|
|
146
|
+
def read(self, off: int, n: int) -> bytes:
|
|
147
|
+
out = b""
|
|
148
|
+
for i, (f, st, sz) in enumerate(zip(self.files, self.starts, self.sizes)):
|
|
149
|
+
if off + n <= st or off >= st + sz:
|
|
150
|
+
continue
|
|
151
|
+
a, b = max(off, st), min(off + n, st + sz)
|
|
152
|
+
fh = self._fh.get(i)
|
|
153
|
+
if fh is None:
|
|
154
|
+
fh = self._fh[i] = open(os.path.join(self.d, f), "rb", buffering=INDEX_BUF)
|
|
155
|
+
fh.seek(a - st)
|
|
156
|
+
out += fh.read(b - a)
|
|
157
|
+
return out
|
|
158
|
+
|
|
159
|
+
def close(self):
|
|
160
|
+
for fh in self._fh.values():
|
|
161
|
+
fh.close()
|
|
162
|
+
self._fh.clear()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def is_archive(d: str) -> bool:
|
|
166
|
+
return os.path.isdir(os.path.join(d, "Pool")) and os.path.isdir(os.path.join(d, "Config"))
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def find_archive_root(path: str) -> str:
|
|
170
|
+
"""Accept the .hbk directory itself, a parent containing one, or the .bkpi file."""
|
|
171
|
+
path = os.path.abspath(os.path.expanduser(path))
|
|
172
|
+
if os.path.isfile(path):
|
|
173
|
+
path = os.path.dirname(path)
|
|
174
|
+
if is_archive(path):
|
|
175
|
+
return path
|
|
176
|
+
if os.path.isdir(path):
|
|
177
|
+
for e in sorted(os.listdir(path)):
|
|
178
|
+
c = os.path.join(path, e)
|
|
179
|
+
if not e.startswith("._") and os.path.isdir(c) and is_archive(c):
|
|
180
|
+
return c
|
|
181
|
+
raise FileNotFoundError(f"no Hyper Backup archive at {path}")
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
# --------------------------------------------------------------------- archive
|
|
185
|
+
|
|
186
|
+
class Archive:
|
|
187
|
+
"""One .hbk archive. Cheap to construct; caches open handles and bucket indexes."""
|
|
188
|
+
|
|
189
|
+
def __init__(self, root: str):
|
|
190
|
+
self.root = find_archive_root(root)
|
|
191
|
+
c, p = f"{self.root}/Config", f"{self.root}/Pool"
|
|
192
|
+
self.vf = Cat(f"{c}/virtual_file.index")
|
|
193
|
+
self.ci = Cat(f"{p}/chunk_index")
|
|
194
|
+
self.fc: dict[int, Cat] = {}
|
|
195
|
+
for e in os.listdir(c):
|
|
196
|
+
m = re.match(r"^file_chunk(\d+)\.index$", e)
|
|
197
|
+
if m and os.path.isdir(os.path.join(c, e)):
|
|
198
|
+
self.fc[int(m.group(1))] = Cat(os.path.join(c, e))
|
|
199
|
+
if not self.fc:
|
|
200
|
+
raise FileNotFoundError(f"no file_chunk<N>.index directories in {c}")
|
|
201
|
+
pools = [e for e in os.listdir(p) if e.isdigit() and os.path.isdir(os.path.join(p, e))]
|
|
202
|
+
if not pools:
|
|
203
|
+
raise FileNotFoundError(f"no numbered pool directory under {p}")
|
|
204
|
+
self.pools = sorted(pools, key=int)
|
|
205
|
+
self.pool = os.path.join(p, self.pools[0])
|
|
206
|
+
self._bidx: OrderedDict[int, bytes] = OrderedDict()
|
|
207
|
+
self._bdat: OrderedDict[int, object] = OrderedDict()
|
|
208
|
+
self._brec: dict[int, int] = {}
|
|
209
|
+
|
|
210
|
+
if self.vf.record_size != 56:
|
|
211
|
+
raise UnsupportedArchive(
|
|
212
|
+
f"virtual_file record size {self.vf.record_size}, only 56 is implemented")
|
|
213
|
+
if self.ci.record_size not in (16, 29):
|
|
214
|
+
raise UnsupportedArchive(
|
|
215
|
+
f"chunk_index record size {self.ci.record_size}, "
|
|
216
|
+
"only 16 (v1/v2) and 29 (v3) are implemented")
|
|
217
|
+
|
|
218
|
+
# -- metadata ---------------------------------------------------------
|
|
219
|
+
|
|
220
|
+
def task_config(self) -> dict:
|
|
221
|
+
out: dict[str, str] = {}
|
|
222
|
+
try:
|
|
223
|
+
p = resolve(self.root, "_Syno_TaskConfig")
|
|
224
|
+
except FileNotFoundError:
|
|
225
|
+
return out
|
|
226
|
+
with open(p, encoding="utf-8", errors="replace") as fh:
|
|
227
|
+
for line in fh:
|
|
228
|
+
if "=" in line and not line.startswith("["):
|
|
229
|
+
k, v = line.split("=", 1)
|
|
230
|
+
out[k.strip()] = v.strip().strip('"')
|
|
231
|
+
return out
|
|
232
|
+
|
|
233
|
+
def is_encrypted(self) -> bool:
|
|
234
|
+
return self.task_config().get("enable_data_encrypt", "false").lower() == "true"
|
|
235
|
+
|
|
236
|
+
def shares(self) -> list[str]:
|
|
237
|
+
d = f"{self.root}/Config/@Share"
|
|
238
|
+
if not os.path.isdir(d):
|
|
239
|
+
return []
|
|
240
|
+
return sorted(e for e in os.listdir(d)
|
|
241
|
+
if not e.startswith("._") and os.path.isdir(os.path.join(d, e)))
|
|
242
|
+
|
|
243
|
+
def share_versions(self, share: str) -> list[int]:
|
|
244
|
+
d = f"{self.root}/Config/@Share/{share}"
|
|
245
|
+
vs = set()
|
|
246
|
+
for f in os.listdir(d):
|
|
247
|
+
if f.startswith("._") or f.startswith("complete_list"):
|
|
248
|
+
continue
|
|
249
|
+
m = re.match(r"^(\d+)\.db(?:\.\d+)?$", f)
|
|
250
|
+
if m:
|
|
251
|
+
vs.add(int(m.group(1)))
|
|
252
|
+
return sorted(vs)
|
|
253
|
+
|
|
254
|
+
def share_db(self, share: str, version: int | None = None) -> str:
|
|
255
|
+
d = f"{self.root}/Config/@Share/{share}"
|
|
256
|
+
if version is None:
|
|
257
|
+
vs = self.share_versions(share)
|
|
258
|
+
if not vs:
|
|
259
|
+
raise FileNotFoundError(f"no version db in {d}")
|
|
260
|
+
version = max(vs)
|
|
261
|
+
return resolve(d, f"{version}.db")
|
|
262
|
+
|
|
263
|
+
# -- chunk plumbing ---------------------------------------------------
|
|
264
|
+
|
|
265
|
+
def _bucket(self, bid: int):
|
|
266
|
+
idx = self._bidx.get(bid)
|
|
267
|
+
if idx is None:
|
|
268
|
+
sub = os.path.join(self.pool, str(bid >> 11))
|
|
269
|
+
f = bid & 0x7FF
|
|
270
|
+
with open(resolve(sub, f"{f}.index"), "rb") as fh:
|
|
271
|
+
idx = fh.read()
|
|
272
|
+
w = struct.unpack(">16I", idx[:64])
|
|
273
|
+
if w[0] != MAGIC:
|
|
274
|
+
raise UnsupportedArchive(f"bucket {bid}: bad index magic {w[0]:#010x}")
|
|
275
|
+
if w[4] not in (28, 32):
|
|
276
|
+
raise UnsupportedArchive(
|
|
277
|
+
f"bucket {bid}: index record size {w[4]}, only 28 and 32 are implemented")
|
|
278
|
+
self._brec[bid] = w[4]
|
|
279
|
+
self._bidx[bid] = idx
|
|
280
|
+
self._bdat[bid] = open(resolve(sub, f"{f}.bucket"), "rb", buffering=BUCKET_BUF)
|
|
281
|
+
while len(self._bidx) > MAX_OPEN_BUCKETS:
|
|
282
|
+
k, _ = self._bidx.popitem(last=False)
|
|
283
|
+
self._bdat.pop(k).close()
|
|
284
|
+
else:
|
|
285
|
+
self._bidx.move_to_end(bid)
|
|
286
|
+
self._bdat.move_to_end(bid)
|
|
287
|
+
return idx, self._bdat[bid]
|
|
288
|
+
|
|
289
|
+
def chunk_ref(self, key: int, depth: int = 0):
|
|
290
|
+
"""Resolve a chunk_index key to (bucket_id, bucket_index_offset)."""
|
|
291
|
+
if depth > 16:
|
|
292
|
+
raise ValueError("chunk_index cite loop")
|
|
293
|
+
n = self.ci.record_size
|
|
294
|
+
r = self.ci.read(key, n)
|
|
295
|
+
if len(r) < min(n, 9):
|
|
296
|
+
raise ValueError(f"short chunk_index record at {key}")
|
|
297
|
+
if n == 16: # v1/v2: bucket_id and offset sit at the front
|
|
298
|
+
return struct.unpack(">ii", r[0:8])
|
|
299
|
+
if r[0] & 1: # v3 bit0 = intra-cite indirection (dedup)
|
|
300
|
+
return self.chunk_ref(struct.unpack(">q", r[1:9])[0], depth + 1)
|
|
301
|
+
return struct.unpack(">ii", r[1:9])
|
|
302
|
+
|
|
303
|
+
def read_chunk(self, bid: int, boff: int, verify: bool = True) -> bytes:
|
|
304
|
+
idx, fh = self._bucket(bid)
|
|
305
|
+
n = self._brec[bid]
|
|
306
|
+
rec = idx[boff:boff + n]
|
|
307
|
+
if len(rec) < n:
|
|
308
|
+
raise ValueError(f"bucket {bid} index truncated at {boff}")
|
|
309
|
+
clen, off, ulen = struct.unpack(">III", rec[:12])
|
|
310
|
+
# 32-byte records carry a trailing CRC32 over the record; 28-byte legacy ones do not
|
|
311
|
+
if verify and n == 32 and (zlib.crc32(rec[:28]) & 0xFFFFFFFF) != struct.unpack(">I", rec[28:32])[0]:
|
|
312
|
+
raise ValueError(f"bucket index CRC32 mismatch b{bid}@{boff}")
|
|
313
|
+
fh.seek(off)
|
|
314
|
+
raw = fh.read(clen)
|
|
315
|
+
dst = ctypes.create_string_buffer(ulen)
|
|
316
|
+
n = lz4().LZ4_decompress_safe(raw, dst, clen, ulen)
|
|
317
|
+
data = dst.raw[:n] if n > 0 else b""
|
|
318
|
+
if n != ulen:
|
|
319
|
+
try:
|
|
320
|
+
data = zlib.decompress(raw) # compress type 4
|
|
321
|
+
except Exception:
|
|
322
|
+
raise ValueError(f"decompress failed b{bid}@{boff} lz4={n} want={ulen}") from None
|
|
323
|
+
if verify and hashlib.md5(data).digest() != rec[12:28]:
|
|
324
|
+
raise ValueError(f"MD5 mismatch b{bid}@{boff}")
|
|
325
|
+
return data
|
|
326
|
+
|
|
327
|
+
def extract(self, ovf: int, size: int, verify: bool = True, out=None):
|
|
328
|
+
"""Rebuild one file. Writes to `out` if given (returns byte count), else returns bytes."""
|
|
329
|
+
if not size:
|
|
330
|
+
return 0 if out is not None else b""
|
|
331
|
+
if ovf is None or ovf < 0:
|
|
332
|
+
# sentinel: the whole file is deduped into Pool/file_pool rather than chunked.
|
|
333
|
+
# Exactly 1 of 501,278 files in the reference archive. Not decoded.
|
|
334
|
+
raise ValueError("whole-file dedup entry (Pool/file_pool) - not supported")
|
|
335
|
+
v = struct.unpack(">q", self.vf.read(ovf, 8))[0]
|
|
336
|
+
fc = self.fc.get(v >> 48)
|
|
337
|
+
if fc is None:
|
|
338
|
+
raise ValueError(f"no file_chunk{v >> 48}.index in archive")
|
|
339
|
+
off = v & 0xFFFFFFFFFFFF
|
|
340
|
+
parts, got, i, BATCH = [], 0, 0, 4096
|
|
341
|
+
while got < size:
|
|
342
|
+
blob = fc.read(off + i * 8, 8 * BATCH)
|
|
343
|
+
if not blob:
|
|
344
|
+
raise ValueError(f"chunk list exhausted at {got}/{size} bytes")
|
|
345
|
+
for j in range(0, len(blob) - 7, 8):
|
|
346
|
+
if got >= size:
|
|
347
|
+
break
|
|
348
|
+
bid, boff = self.chunk_ref(struct.unpack(">q", blob[j:j + 8])[0])
|
|
349
|
+
d = self.read_chunk(bid, boff, verify=verify)
|
|
350
|
+
got += len(d)
|
|
351
|
+
if out is not None:
|
|
352
|
+
out.write(d)
|
|
353
|
+
else:
|
|
354
|
+
parts.append(d)
|
|
355
|
+
i += 1
|
|
356
|
+
if got != size:
|
|
357
|
+
raise ValueError(f"size mismatch: rebuilt {got}, expected {size}")
|
|
358
|
+
return got if out is not None else b"".join(parts)
|
|
359
|
+
|
|
360
|
+
def close(self):
|
|
361
|
+
self.vf.close()
|
|
362
|
+
self.ci.close()
|
|
363
|
+
for c in self.fc.values():
|
|
364
|
+
c.close()
|
|
365
|
+
for fh in self._bdat.values():
|
|
366
|
+
fh.close()
|
|
367
|
+
self._bdat.clear()
|
|
368
|
+
self._bidx.clear()
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
# ------------------------------------------------- module-level convenience API
|
|
372
|
+
# One cached Archive per root, so worker processes build it once and reuse it.
|
|
373
|
+
|
|
374
|
+
_cache: dict[str, Archive] = {}
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def open_archive(root: str | None = None) -> Archive:
|
|
378
|
+
root = root or os.environ.get("HBK_ROOT") or ""
|
|
379
|
+
key = os.path.abspath(os.path.expanduser(root))
|
|
380
|
+
a = _cache.get(key)
|
|
381
|
+
if a is None:
|
|
382
|
+
a = _cache[key] = Archive(root)
|
|
383
|
+
return a
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def extract(ovf: int, size: int, verify: bool = True, out=None, root: str | None = None):
|
|
387
|
+
return open_archive(root).extract(ovf, size, verify=verify, out=out)
|
hbkit/cli.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Recover files from a Synology Hyper Backup (.hbk) archive. No Synology software needed.
|
|
3
|
+
|
|
4
|
+
hbk <archive> doctor check whether it can be recovered
|
|
5
|
+
hbk <archive> info show task, shares, codec
|
|
6
|
+
hbk <archive> list [pattern] search the file index
|
|
7
|
+
hbk <archive> get <glob> <dest> [-j N] extract (keeps tree + mtimes)
|
|
8
|
+
hbk <archive> verify <glob> [-j N] integrity-check, write nothing
|
|
9
|
+
hbk <archive> tui browse in the full-screen UI
|
|
10
|
+
|
|
11
|
+
<archive> is a .hbk directory, or any drive/folder containing one.
|
|
12
|
+
Globs match the full archive path, which starts with the share name.
|
|
13
|
+
Extraction is resumable: correctly-sized files are skipped.
|
|
14
|
+
|
|
15
|
+
hbk /Volumes/Backup doctor
|
|
16
|
+
hbk /Volumes/Backup list holiday
|
|
17
|
+
hbk /Volumes/Backup get "/Photos/2019/*" ~/restore
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import fnmatch
|
|
22
|
+
import os
|
|
23
|
+
import shutil
|
|
24
|
+
import sqlite3
|
|
25
|
+
import sys
|
|
26
|
+
import time
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
from . import archive as hbk
|
|
30
|
+
from . import index as hbk_index
|
|
31
|
+
from . import runner as hbk_run
|
|
32
|
+
|
|
33
|
+
CODECS = {"0": "none", "1": "lz4", "2": "lz4-hc", "4": "zlib"}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def human(n) -> str:
|
|
37
|
+
n = float(n or 0)
|
|
38
|
+
for u in ("B", "K", "M", "G", "T"):
|
|
39
|
+
if abs(n) < 1024 or u == "T":
|
|
40
|
+
return f"{n:.0f}B" if u == "B" else f"{n:,.1f}{u}"
|
|
41
|
+
n /= 1024
|
|
42
|
+
return f"{n:,.1f}T"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def hms(s) -> str:
|
|
46
|
+
s = max(0, int(s or 0))
|
|
47
|
+
h, m, sec = s // 3600, (s % 3600) // 60, s % 60
|
|
48
|
+
return f"{h}:{m:02d}:{sec:02d}" if h else f"{m:02d}:{sec:02d}"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def all_files(db):
|
|
52
|
+
"""(path, ovf, size, mtime) for every file; path begins with the share name."""
|
|
53
|
+
return db.execute("""
|
|
54
|
+
WITH RECURSIVE t(id,path,isdir) AS (
|
|
55
|
+
SELECT id, '/'||name, isdir FROM node WHERE parent IS NULL
|
|
56
|
+
UNION ALL
|
|
57
|
+
SELECT n.id, t.path||'/'||n.name, n.isdir
|
|
58
|
+
FROM node n JOIN t ON n.parent=t.id
|
|
59
|
+
)
|
|
60
|
+
SELECT t.path, n.ovf, n.size, n.mtime
|
|
61
|
+
FROM t JOIN node n ON n.id=t.id
|
|
62
|
+
WHERE t.isdir=0 AND n.ovf IS NOT NULL
|
|
63
|
+
""")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def open_index(archive):
|
|
67
|
+
arc = hbk.Archive(archive)
|
|
68
|
+
if arc.is_encrypted():
|
|
69
|
+
sys.exit("archive is encrypted - extraction is not supported")
|
|
70
|
+
if not hbk_index.is_current(arc):
|
|
71
|
+
print("indexing archive (first use only)...", file=sys.stderr)
|
|
72
|
+
hbk_index.build(arc, progress=lambda s, d, n: print(f" {s} [{d}/{n}]", file=sys.stderr))
|
|
73
|
+
return arc, sqlite3.connect(f"file:{hbk_index.cache_path(arc)}?mode=ro", uri=True)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def run(arc, sel, dest, jobs, check):
|
|
77
|
+
total = sum(s[2] for s in sel)
|
|
78
|
+
print(f"{len(sel):,} files, {human(total)}, {jobs} workers"
|
|
79
|
+
+ (" - verify only, nothing written" if check else f" -> {dest}"))
|
|
80
|
+
if not check:
|
|
81
|
+
os.makedirs(dest, exist_ok=True)
|
|
82
|
+
r = hbk_run.Runner(arc.root, dest, sel, jobs, verify=True, check=check)
|
|
83
|
+
tty = sys.stderr.isatty()
|
|
84
|
+
t0 = time.time()
|
|
85
|
+
r.start()
|
|
86
|
+
try:
|
|
87
|
+
while not r.finished:
|
|
88
|
+
r.poll()
|
|
89
|
+
el = max(time.time() - t0, 1e-6)
|
|
90
|
+
pct = 100 * r.bytes_done / total if total else 100
|
|
91
|
+
rate = r.bytes_done / el
|
|
92
|
+
eta = (total - r.bytes_done) / rate if rate > 0 else 0
|
|
93
|
+
if tty: # only paint a live line on a terminal; piped output stays clean
|
|
94
|
+
sys.stderr.write(f"\r {pct:5.1f}% {r.done_files:,}/{r.total_files:,} "
|
|
95
|
+
f"{human(r.bytes_done)} {rate/1e6:,.0f} MB/s eta {hms(eta)} ")
|
|
96
|
+
sys.stderr.flush()
|
|
97
|
+
time.sleep(0.25)
|
|
98
|
+
r.poll()
|
|
99
|
+
except KeyboardInterrupt:
|
|
100
|
+
r.cancel()
|
|
101
|
+
print("\ncancelled", file=sys.stderr)
|
|
102
|
+
finally:
|
|
103
|
+
r.cleanup()
|
|
104
|
+
el = max(time.time() - t0, 1e-6)
|
|
105
|
+
if tty:
|
|
106
|
+
sys.stderr.write("\r" + " " * 78 + "\r")
|
|
107
|
+
verb = "verified" if check else "extracted"
|
|
108
|
+
print(f"{r.ok:,} {verb}, {r.skipped:,} already present, {r.failed:,} failed "
|
|
109
|
+
f"in {hms(el)} ({r.bytes_done/el/1e6:,.0f} MB/s)")
|
|
110
|
+
for p, m in r.errors[:20]:
|
|
111
|
+
print(f" FAIL {p}: {m}")
|
|
112
|
+
if len(r.errors) > 20:
|
|
113
|
+
print(f" ... and {len(r.errors)-20} more")
|
|
114
|
+
return 1 if r.failed else 0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def main() -> int:
|
|
118
|
+
a = sys.argv[1:]
|
|
119
|
+
jobs = 8
|
|
120
|
+
if "-j" in a:
|
|
121
|
+
i = a.index("-j")
|
|
122
|
+
jobs = int(a[i + 1])
|
|
123
|
+
del a[i:i + 2]
|
|
124
|
+
if len(a) < 2:
|
|
125
|
+
print(__doc__)
|
|
126
|
+
return 1
|
|
127
|
+
archive, cmd = a[0], a[1]
|
|
128
|
+
|
|
129
|
+
if cmd == "doctor":
|
|
130
|
+
from . import doctor
|
|
131
|
+
r = doctor.diagnose(archive)
|
|
132
|
+
print(doctor.render(r))
|
|
133
|
+
return 0 if (r.ok and not r.blockers) else 1
|
|
134
|
+
|
|
135
|
+
if cmd == "tui":
|
|
136
|
+
from .tui import HBKApp
|
|
137
|
+
HBKApp(jobs, archive).run()
|
|
138
|
+
return 0
|
|
139
|
+
|
|
140
|
+
if cmd == "info":
|
|
141
|
+
arc = hbk.Archive(archive)
|
|
142
|
+
cfg = arc.task_config()
|
|
143
|
+
print(f"archive : {arc.root}")
|
|
144
|
+
print(f"task : {cfg.get('name','?')} host {cfg.get('host_name','?')}")
|
|
145
|
+
print(f"codec : {CODECS.get(cfg.get('data_compress_type',''), '?')}")
|
|
146
|
+
print(f"encrypted : {arc.is_encrypted()}")
|
|
147
|
+
print(f"sources : {cfg.get('backup_folders','?')}")
|
|
148
|
+
for s in arc.shares():
|
|
149
|
+
print(f" share {s} versions={arc.share_versions(s)}")
|
|
150
|
+
return 0
|
|
151
|
+
|
|
152
|
+
arc, db = open_index(archive)
|
|
153
|
+
|
|
154
|
+
if cmd == "list":
|
|
155
|
+
pat = a[2].lower() if len(a) > 2 else ""
|
|
156
|
+
n = tot = 0
|
|
157
|
+
for path, ovf, size, mt in all_files(db):
|
|
158
|
+
if pat in path.lower():
|
|
159
|
+
n += 1
|
|
160
|
+
tot += size or 0
|
|
161
|
+
if n <= 200:
|
|
162
|
+
print(f"{size or 0:>13,} {path}")
|
|
163
|
+
print(f"\n{n:,} files, {human(tot)}" + (" (first 200 shown)" if n > 200 else ""))
|
|
164
|
+
return 0
|
|
165
|
+
|
|
166
|
+
if cmd in ("get", "verify"):
|
|
167
|
+
if len(a) < 3 or (cmd == "get" and len(a) < 4):
|
|
168
|
+
print(__doc__)
|
|
169
|
+
return 1
|
|
170
|
+
glob = a[2]
|
|
171
|
+
dest = os.path.abspath(os.path.expanduser(a[3])) if cmd == "get" else ""
|
|
172
|
+
sel = [row for row in all_files(db) if fnmatch.fnmatch(row[0], glob)]
|
|
173
|
+
if not sel:
|
|
174
|
+
print("no files matched that glob")
|
|
175
|
+
return 1
|
|
176
|
+
if cmd == "get":
|
|
177
|
+
need = sum(s[2] for s in sel)
|
|
178
|
+
probe = dest
|
|
179
|
+
while probe and not os.path.exists(probe):
|
|
180
|
+
probe = os.path.dirname(probe)
|
|
181
|
+
free = shutil.disk_usage(probe or "/").free
|
|
182
|
+
if need > free:
|
|
183
|
+
print(f"not enough space: need {human(need)}, have {human(free)} free")
|
|
184
|
+
return 1
|
|
185
|
+
return run(arc, sel, dest, jobs, cmd == "verify")
|
|
186
|
+
|
|
187
|
+
print(__doc__)
|
|
188
|
+
return 1
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def main_entry() -> int:
|
|
192
|
+
"""Console-script entry point; keeps `main()` usable as a plain function."""
|
|
193
|
+
try:
|
|
194
|
+
return main()
|
|
195
|
+
except KeyboardInterrupt:
|
|
196
|
+
return 130
|
|
197
|
+
except hbk.UnsupportedArchive as e:
|
|
198
|
+
print(f"unsupported archive: {e}", file=sys.stderr)
|
|
199
|
+
return 2
|
|
200
|
+
except FileNotFoundError as e:
|
|
201
|
+
print(f"{e}", file=sys.stderr)
|
|
202
|
+
return 2
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
if __name__ == "__main__":
|
|
206
|
+
sys.exit(main_entry())
|