checkdisk 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checkdisk.py ADDED
@@ -0,0 +1,4328 @@
1
+ '''A self-contained, pure-Python NTFS repair tool — the parts of chkdsk /f that
2
+ matter for a crashed volume, with no dependency beyond the standard library.
3
+
4
+ A crash can leave a directory's B+ tree ($I30 index) holding entries whose MFT
5
+ record is dead or reused (sequence-number mismatch). The ntfs3 kernel driver then
6
+ returns ESTALE/EINVAL on those names forever, and `ntfsfix` cannot repair indexes.
7
+ Windows chkdsk fixes this in stage 2 by validating each index entry against the
8
+ MFT and deleting the dangling ones. This tool does that and the other repairs
9
+ below with its own native read/write engine: it parses and rewrites the on-disk
10
+ NTFS structures directly (records, runlists, B+ tree splits/merges, $Bitmap,
11
+ $Secure, the USN journal), with multi-sector fixups and plan-then-commit
12
+ atomicity.
13
+
14
+ Commands (device = partition or image file; dir = path INSIDE the volume):
15
+ scan <device> <dir> read-only: every entry health-checked
16
+ inspect <device> <dir> <name> read-only triage: free/reused/orphan?
17
+ rm <device> <dir> <name> [--really] drop one dangling entry
18
+ purge <device> <dir> <name> [--really] true orphan: free dirent+record+clusters
19
+ rebuild <device> <dir> [--really] torn INDX: rebuild the whole index
20
+ from the children's MFT records
21
+ fix|/f <device> [--really] chkdsk /f flow — stage 1: verify MFT
22
+ records + attribute runlists (repairs
23
+ torn empty-attribute truncates);
24
+ stage 2: walk every directory index,
25
+ repair dangling entries, rebuild torn
26
+ indexes; stage 5: reconcile $Bitmap
27
+ cluster accounting
28
+ backup <device> <outdir> [dir ...] dump boot sectors, $MFT, $MFTMirr and
29
+ the given directories' raw $I30 indexes
30
+ (targeted pre-repair backup)
31
+ usn <device> [--really] [--reset] validate the USN change journal
32
+ ($UsnJrnl: $Max sanity + full $J record
33
+ walk); --really resets a corrupt
34
+ journal, --reset forces the reset
35
+ secure <device> [--really] chkdsk stage 3: validate $Secure —
36
+ every $SDS descriptor (header, hash,
37
+ 256KiB mirror), the $SII/$SDH indexes
38
+ against $SDS, and every file's
39
+ security_id; --really repairs mirrors
40
+ and rebuilds the indexes
41
+ surface <device> chkdsk stage 4, report-only: read every
42
+ cluster, attribute unreadable ones to
43
+ their owning files (also available as
44
+ `/r` or `fix --surface`)
45
+ All writing commands are dry-run unless --really is provided.
46
+ '''
47
+
48
+ from __future__ import annotations
49
+
50
+ import argparse, array, bisect, errno, hashlib, os, struct, subprocess, sys, time
51
+
52
+
53
+ __version__ = '0.1.0'
54
+
55
+
56
+ # Positioned I/O. os.pread/os.pwrite are POSIX-only; on Windows fall back to
57
+ # lseek + read/write — the tool is single-threaded, so the atomicity a real
58
+ # pread would give over a separate seek is moot here. _O_BINARY keeps Windows
59
+ # from CRLF/EOF-translating a raw disk image (it is 0, a no-op, on POSIX).
60
+ _O_BINARY = getattr(os, 'O_BINARY', 0)
61
+ if hasattr(os, 'pread'):
62
+ _pread, _pwrite = os.pread, os.pwrite
63
+ else:
64
+ def _pread(fd, n, offset):
65
+ os.lseek(fd, offset, os.SEEK_SET)
66
+ return os.read(fd, n)
67
+
68
+ def _pwrite(fd, data, offset):
69
+ os.lseek(fd, offset, os.SEEK_SET)
70
+ return os.write(fd, data)
71
+
72
+
73
+ MREF_MASK = (1 << 48) - 1 # low 48 bits = record number, high 16 = sequence
74
+
75
+
76
+ # UTF-16LE attribute/stream names (plain bytes; the reader matches names
77
+ # as bytes).
78
+ INDEX_I30 = '$I30'.encode('utf-16-le')
79
+ USN_MAX_NAME = '$Max'.encode('utf-16-le')
80
+ USN_J_NAME = '$J'.encode('utf-16-le')
81
+ SDS_NAME = '$SDS'.encode('utf-16-le')
82
+ SII_NAME = '$SII'.encode('utf-16-le')
83
+ SDH_NAME = '$SDH'.encode('utf-16-le')
84
+ R_NAME = '$R'.encode('utf-16-le')
85
+ O_NAME = '$O'.encode('utf-16-le')
86
+
87
+ SDS_BLOCK = 0x40000 # $SDS entries mirror in alternating 256 KiB blocks
88
+
89
+ AT_FILE_NAME = 0x30
90
+ AT_DATA = 0x80
91
+ AT_INDEX_ROOT = 0x90
92
+ AT_INDEX_ALLOCATION = 0xA0
93
+ AT_BITMAP = 0xB0
94
+ AT_STANDARD_INFORMATION = 0x10
95
+ AT_ATTRIBUTE_LIST = 0x20
96
+ FILE_ROOT = 5
97
+ FILE_BITMAP = 6
98
+ FILE_LOGFILE = 2
99
+ FILE_SECURE = 9
100
+ FILE_UPCASE = 10
101
+
102
+ AT_UNNAMED = None
103
+
104
+ NTFS_DT_DIR = 4 # readdir dt_type for a directory (mirrors DT_DIR)
105
+
106
+
107
+ class NtfsError(OSError):
108
+ pass
109
+
110
+
111
+ # ── raw MFT record helpers (read-only parsing for the rebuild census) ────────
112
+
113
+ def _apply_fixups(rec: bytearray) -> bool:
114
+ '''Undo the multi-sector transfer protection; False = the record is torn '''
115
+
116
+ usa_ofs = struct.unpack_from('<H', rec, 4)[0]
117
+ usa_count = struct.unpack_from('<H', rec, 6)[0]
118
+ if not usa_ofs or usa_ofs + 2 * usa_count > len(rec):
119
+ return False
120
+ usn = bytes(rec[usa_ofs:usa_ofs + 2])
121
+ for i in range(1, usa_count):
122
+ end = i * 512
123
+ if end > len(rec) or bytes(rec[end - 2:end]) != usn:
124
+ return False
125
+ rec[end - 2:end] = rec[usa_ofs + 2 * i:usa_ofs + 2 * i + 2]
126
+ return True
127
+
128
+
129
+ def _attrs(rec):
130
+ '''Yield (offset, attr_type, attr_len) along a fixed-up record's attribute
131
+ chain, stopping silently at the 0xFFFFFFFF terminator or any malformed
132
+ header — the shared walk. The one caller that must REPORT a broken
133
+ chain rather than stop (mft_survey) keeps its own loop.'''
134
+ offset = struct.unpack_from('<H', rec, 20)[0]
135
+ while offset + 8 <= len(rec):
136
+ attr_type = struct.unpack_from('<I', rec, offset)[0]
137
+ if attr_type == 0xFFFFFFFF:
138
+ return
139
+ length = struct.unpack_from('<I', rec, offset + 4)[0]
140
+ if length < 24 or offset + length > len(rec):
141
+ return
142
+ yield offset, attr_type, length
143
+ offset += length
144
+
145
+
146
+ def _has_attr(rec: bytes, want: int) -> bool:
147
+ '''True if the fixed-up MFT record carries an attribute of this type '''
148
+ return any(t == want for _off, t, _len in _attrs(rec))
149
+
150
+
151
+ def _seal_fixups(rec: bytearray) -> None:
152
+ '''Inverse of _apply_fixups: bump the update sequence number, stash each
153
+ sector's real last word into the USA, and stamp the USN over the
154
+ sector ends — the multi-sector transfer protection, applied.'''
155
+ usa_ofs = struct.unpack_from('<H', rec, 4)[0]
156
+ usa_count = struct.unpack_from('<H', rec, 6)[0]
157
+ usn = (struct.unpack_from('<H', rec, usa_ofs)[0] + 1) & 0xFFFF or 1
158
+ struct.pack_into('<H', rec, usa_ofs, usn)
159
+ for i in range(1, usa_count):
160
+ end = i * 512
161
+ rec[usa_ofs + 2 * i:usa_ofs + 2 * i + 2] = rec[end - 2:end]
162
+ struct.pack_into('<H', rec, end - 2, usn)
163
+
164
+
165
+ def _file_name_attrs(rec: bytes):
166
+ '''Yield each resident $FILE_NAME attribute value in a fixed-up MFT record '''
167
+ for offset, attr_type, _length in _attrs(rec):
168
+ if attr_type == AT_FILE_NAME and rec[offset + 8] == 0: # resident only
169
+ value_len = struct.unpack_from('<I', rec, offset + 16)[0]
170
+ value_ofs = struct.unpack_from('<H', rec, offset + 20)[0]
171
+ fn = rec[offset + value_ofs:offset + value_ofs + value_len]
172
+ if len(fn) >= 66 and len(fn) >= 66 + 2 * fn[64]:
173
+ yield fn
174
+
175
+
176
+ def _check_mapping_pairs(rec: bytes, attr_off: int, length: int,
177
+ nr_clusters: int) -> tuple[str | None, bool, list]:
178
+ '''Validate and decode a non-resident attribute's mapping pairs: sane pair
179
+ headers, no overrun, positive run lengths,
180
+ LCNs inside the volume, total clusters matching the highest_vcn header.
181
+ Returns (problem, is_phantom_empty, runs) — runs is the decoded
182
+ [(lcn, run_len, vcn), ...] for allocated (non-sparse) runs, the shared
183
+ source for the stage-5 used-cluster map; is_phantom_empty marks the one
184
+ safely auto-repairable case: a header that says "empty" (highest_vcn
185
+ == -1) while well-formed pairs still describe runs (torn truncate).'''
186
+
187
+ lowest = struct.unpack_from('<q', rec, attr_off + 16)[0]
188
+ highest = struct.unpack_from('<q', rec, attr_off + 24)[0]
189
+ mp_ofs = struct.unpack_from('<H', rec, attr_off + 32)[0]
190
+ end = attr_off + length
191
+ pos = attr_off + mp_ofs
192
+ out: list[tuple[int, int]] = []
193
+ if pos >= end:
194
+ return f'mapping pairs offset {mp_ofs} outside attribute', False, out
195
+ clusters, lcn, vcn = 0, 0, lowest
196
+ while pos < end and rec[pos]:
197
+ header = rec[pos]
198
+ len_sz, ofs_sz = header & 0xF, header >> 4
199
+ if not 1 <= len_sz <= 8 or ofs_sz > 8:
200
+ return f'bad pair header {header:#x} at +{pos - attr_off}', False, out
201
+ if pos + 1 + len_sz + ofs_sz > end:
202
+ return f'pair overruns attribute at +{pos - attr_off}', False, out
203
+ run_len = int.from_bytes(rec[pos + 1:pos + 1 + len_sz], 'little',
204
+ signed=True)
205
+ if run_len <= 0:
206
+ return f'non-positive run length {run_len} at +{pos - attr_off}', False, out
207
+ if ofs_sz: # 0 = sparse run, lcn unchanged
208
+ lcn += int.from_bytes(rec[pos + 1 + len_sz:pos + 1 + len_sz + ofs_sz],
209
+ 'little', signed=True)
210
+ if lcn < 0:
211
+ return f'negative LCN {lcn} at +{pos - attr_off}', False, out
212
+ if lcn + run_len > nr_clusters:
213
+ return (f'run [{lcn}, {lcn + run_len}) beyond volume '
214
+ f'({nr_clusters} clusters)', False, out)
215
+ out.append((lcn, run_len, vcn))
216
+ clusters += run_len
217
+ vcn += run_len
218
+ pos += 1 + len_sz + ofs_sz
219
+ if pos >= end:
220
+ return 'runlist not terminated inside attribute', False, out
221
+ if highest == -1 and lowest == 0:
222
+ if clusters:
223
+ return (f'{len(out) or 1} phantom run(s) on an empty attribute '
224
+ '(torn truncate)', True, out)
225
+ elif highest >= lowest >= 0 and clusters != highest - lowest + 1:
226
+ return (f'runlist covers {clusters} clusters, '
227
+ f'header says {highest - lowest + 1}'), False, out
228
+ return None, False, out
229
+
230
+
231
+ def _min_bytes_signed(v: int) -> bytes:
232
+ n = 1
233
+ while not -(1 << (8 * n - 1)) <= v < (1 << (8 * n - 1)):
234
+ n += 1
235
+ return v.to_bytes(n, 'little', signed=True)
236
+
237
+
238
+ def _encode_mapping_pairs(runs) -> bytes:
239
+ '''Encode VCN-contiguous [(lcn, length), ...] into NTFS mapping pairs —
240
+ the inverse of _check_mapping_pairs' decode. lcn < 0 marks a sparse run
241
+ (the offset field is omitted). Terminated by a 0 header byte.'''
242
+ out = bytearray()
243
+ prev = 0
244
+ for lcn, length in runs:
245
+ if length <= 0:
246
+ raise NtfsError(0, f'non-positive run length {length}')
247
+ # both fields are signed varints: a length of 128 must be encoded as
248
+ # 80 00 (two bytes) — the driver sign-extends the top byte and treats
249
+ # a lone 80 as -128, refusing the whole runlist
250
+ len_bytes = _min_bytes_signed(length)
251
+ if lcn < 0: # sparse
252
+ out.append(len(len_bytes))
253
+ out += len_bytes
254
+ else:
255
+ off_bytes = _min_bytes_signed(lcn - prev)
256
+ out.append((len(off_bytes) << 4) | len(len_bytes))
257
+ out += len_bytes + off_bytes
258
+ prev = lcn
259
+ out.append(0)
260
+ return bytes(out)
261
+
262
+
263
+ def _walk_usn_records(read, start: int, end: int) -> tuple[int, list[str]]:
264
+ '''Validate the USN $J stream in [start, end): every record's length is
265
+ sane and 8-aligned, its Usn field equals its own byte offset, names stay
266
+ inside the record, and zero fill appears only as tail padding up to the
267
+ next 4 KiB page boundary. `read(pos, count) -> bytes`. Returns
268
+ (records_walked, problems); stops at the first structural problem — a
269
+ corrupt journal gets reset wholesale, so one finding is enough.'''
270
+
271
+ problems: list[str] = []
272
+ count, pos = 0, start
273
+ if pos & 7:
274
+ return 0, [f'journal start {pos} not 8-byte aligned']
275
+ while pos < end and not problems:
276
+ chunk = read(pos, min(1 << 20, end - pos))
277
+ if not chunk:
278
+ problems.append(f'$J read failed at usn {pos}')
279
+ break
280
+ off, n = 0, len(chunk)
281
+ while off + 4 <= n and not problems:
282
+ reclen = struct.unpack_from('<I', chunk, off)[0]
283
+ if reclen == 0:
284
+ # zero fill is only valid up to the next 4 KiB page boundary
285
+ nxt = min((pos + off + 4096) & ~4095, end) - pos
286
+ if chunk[off:min(nxt, n)].strip(b'\0'):
287
+ problems.append(f'garbage inside zero padding at usn {pos + off}')
288
+ off = min(nxt, n)
289
+ continue
290
+ if reclen & 7 or not 60 <= reclen <= 8192:
291
+ problems.append(f'bad record length {reclen} at usn {pos + off}')
292
+ break
293
+ if off + reclen > n:
294
+ break # record crosses the chunk edge — refill from here
295
+ major = struct.unpack_from('<H', chunk, off + 4)[0]
296
+ if major not in (2, 3, 4):
297
+ problems.append(f'unknown USN record version {major} at usn {pos + off}')
298
+ break
299
+ usn_field = {2: 24, 3: 40}.get(major) # v4 carries no Usn to cross-check
300
+ if usn_field is not None:
301
+ claimed = struct.unpack_from('<q', chunk, off + usn_field)[0]
302
+ if claimed != pos + off:
303
+ problems.append(f'record at offset {pos + off} claims usn {claimed}')
304
+ break
305
+ if major == 2:
306
+ nlen, nofs = struct.unpack_from('<HH', chunk, off + 56)
307
+ if nofs + nlen > reclen:
308
+ problems.append(f'file name overruns record at usn {pos + off}')
309
+ break
310
+ count += 1
311
+ off += reclen
312
+ if problems:
313
+ break
314
+ if off == 0:
315
+ problems.append(f'truncated record at usn {pos} (overruns journal end)')
316
+ break
317
+ pos += off
318
+ return count, problems
319
+
320
+
321
+ def _security_hash(descriptor: bytes) -> int:
322
+ '''The $Secure descriptor hash: over each little-endian u32 word,
323
+ hash = word + rol32(hash, 3). Trailing 1-3 bytes are ignored.'''
324
+ h = 0
325
+ for (word,) in struct.iter_unpack('<I', descriptor[:len(descriptor) & ~3]):
326
+ h = (word + ((h << 3) | (h >> 29))) & 0xFFFFFFFF
327
+ return h
328
+
329
+
330
+ def _parse_sds(sds: bytes) -> tuple[dict, list[str], list[tuple]]:
331
+ '''Walk $SDS: entries live in even 256 KiB blocks, each mirrored into the
332
+ following odd block. Returns ({security_id: (hash, offset, length)},
333
+ problems, mirror_fixes) where each mirror fix is (offset, length, source)
334
+ with source 'primary' or 'mirror' — the side whose hash verifies.'''
335
+
336
+ entries: dict[int, tuple] = {}
337
+ problems: list[str] = []
338
+ fixes: list[tuple] = []
339
+ pos, end = 0, len(sds)
340
+ while pos + 20 <= end:
341
+ block = pos // SDS_BLOCK
342
+ if block & 1: # mirror region — validated alongside its primary
343
+ pos = (block + 1) * SDS_BLOCK
344
+ continue
345
+ hash_, sid, off, length = struct.unpack_from('<IIQI', sds, pos)
346
+ if length == 0: # end of entries in this block
347
+ pos = (block + 2) * SDS_BLOCK
348
+ continue
349
+ if length < 20 or pos + length > min(end, (block + 1) * SDS_BLOCK):
350
+ problems.append(f'bad $SDS entry length {length} at offset {pos}')
351
+ break
352
+ if off != pos:
353
+ problems.append(f'$SDS entry at {pos} claims offset {off}')
354
+ break
355
+ primary_ok = _security_hash(sds[pos + 20:pos + length]) == hash_
356
+ mirror_pos = pos + SDS_BLOCK
357
+ mirror_ok = None
358
+ if mirror_pos + length <= end:
359
+ mirror = sds[mirror_pos:mirror_pos + length]
360
+ # a faithful mirror matches byte-for-byte; tolerate an offset field
361
+ # that points at itself instead of the primary
362
+ same = (mirror[:8] == sds[pos:pos + 8]
363
+ and mirror[16:] == sds[pos + 16:pos + length]
364
+ and struct.unpack_from('<Q', mirror, 8)[0] in (pos, mirror_pos))
365
+ if same:
366
+ mirror_ok = True
367
+ else:
368
+ mhash = struct.unpack_from('<I', mirror, 0)[0]
369
+ mirror_ok = (_security_hash(mirror[20:]) == mhash
370
+ and struct.unpack_from('<I', mirror, 4)[0] == sid)
371
+ if primary_ok:
372
+ fixes.append((pos, length, 'primary'))
373
+ elif mirror_ok:
374
+ fixes.append((pos, length, 'mirror'))
375
+ else:
376
+ problems.append(f'$SDS entry id {sid} at {pos}: both copies corrupt')
377
+ if not primary_ok and mirror_ok is not True:
378
+ if mirror_ok is None:
379
+ problems.append(f'$SDS entry id {sid} at {pos}: hash mismatch, no mirror')
380
+ if sid in entries:
381
+ problems.append(f'duplicate security id {sid} in $SDS')
382
+ entries[sid] = (hash_, pos, length)
383
+ pos = (pos + length + 15) & ~15
384
+ return entries, problems, fixes
385
+
386
+
387
+ def _walk_index_node(buf: bytes, hdr_off: int, entries: list, problems: list,
388
+ label: str, dir_index: bool = False, key_fn=None) -> None:
389
+ '''Collect (key, data) from one index node whose INDEX_HEADER sits at
390
+ hdr_off — works for $INDEX_ROOT values and fixed-up INDX blocks alike.'''
391
+ if hdr_off + 16 > len(buf):
392
+ problems.append(f'{label}: truncated index header')
393
+ return
394
+ prev = None
395
+ entries_ofs, index_len = struct.unpack_from('<II', buf, hdr_off)
396
+ pos = hdr_off + entries_ofs
397
+ limit = min(hdr_off + index_len, len(buf))
398
+ while pos + 16 <= limit:
399
+ data_ofs, data_len = struct.unpack_from('<HH', buf, pos)
400
+ length, key_len, flags = struct.unpack_from('<HHH', buf, pos + 8)
401
+ if length < 16 or pos + length > limit:
402
+ problems.append(f'{label}: bad index entry length {length} at +{pos}')
403
+ return
404
+ if not flags & 2: # not the END entry
405
+ if key_fn is not None and 16 + key_len <= length:
406
+ cur = key_fn(buf[pos + 16:pos + 16 + key_len])
407
+ if prev is not None and cur < prev:
408
+ problems.append(f'{label}: keys out of order at +{pos}')
409
+ prev = cur
410
+ if dir_index:
411
+ if 16 + key_len > length:
412
+ problems.append(f'{label}: entry bounds corrupt at +{pos}')
413
+ return
414
+ entries.append((buf[pos + 16:pos + 16 + key_len],
415
+ struct.unpack_from('<Q', buf, pos)[0]))
416
+ else:
417
+ if 16 + key_len > length or data_ofs + data_len > length:
418
+ problems.append(f'{label}: entry bounds corrupt at +{pos}')
419
+ return
420
+ entries.append((buf[pos + 16:pos + 16 + key_len],
421
+ buf[pos + data_ofs:pos + data_ofs + data_len]))
422
+ if flags & 2:
423
+ return
424
+ pos += length
425
+ problems.append(f'{label}: node not terminated by an END entry')
426
+
427
+
428
+ def _view_index_entries(root: bytes, alloc: bytes | None, bitmap: bytes | None,
429
+ label: str, dir_index: bool = False,
430
+ key_fn=None) -> tuple[list, list]:
431
+ '''All (key, data) pairs of a view index: the $INDEX_ROOT node plus every
432
+ in-use, fixup-verified INDX block of $INDEX_ALLOCATION.'''
433
+ entries: list[tuple[bytes, bytes]] = []
434
+ problems: list[str] = []
435
+ if len(root) < 32:
436
+ return [], [f'{label}: $INDEX_ROOT too small ({len(root)} bytes)']
437
+ _walk_index_node(root, 16, entries, problems, f'{label} root', dir_index, key_fn)
438
+ if alloc:
439
+ block_size = struct.unpack_from('<I', root, 8)[0]
440
+ if block_size not in (512, 1024, 2048, 4096, 8192):
441
+ return entries, [f'{label}: implausible index block size {block_size}']
442
+ for i in range(len(alloc) // block_size):
443
+ if bitmap is not None and not (i >> 3) < len(bitmap):
444
+ break
445
+ if bitmap is not None and not bitmap[i >> 3] & (1 << (i & 7)):
446
+ continue # free block — stale content is fine
447
+ blk = bytearray(alloc[i * block_size:(i + 1) * block_size])
448
+ if blk[:4] != b'INDX':
449
+ problems.append(f'{label} block {i}: bad magic')
450
+ continue
451
+ if not _apply_fixups(blk):
452
+ problems.append(f'{label} block {i}: torn (fixup mismatch)')
453
+ continue
454
+ _walk_index_node(bytes(blk), 24, entries, problems,
455
+ f'{label} block {i}', dir_index, key_fn)
456
+ return entries, problems
457
+
458
+
459
+ def _surface_scan(pread, end: int, cluster_size: int, progress=None) -> list[int]:
460
+ '''Read [0, end) sequentially in large chunks; when a chunk read fails,
461
+ bisect it down to single clusters. pread(pos, count) -> bytes and raises
462
+ OSError on unreadable media. Returns sorted unreadable cluster numbers.'''
463
+
464
+ bad: list[int] = []
465
+
466
+ def probe(pos: int, length: int) -> None:
467
+ if length <= cluster_size:
468
+ try:
469
+ pread(pos, length)
470
+ except OSError:
471
+ bad.append(pos // cluster_size)
472
+ return
473
+ half = (length // 2) // cluster_size * cluster_size or cluster_size
474
+ for p, n in ((pos, half), (pos + half, length - half)):
475
+ if n <= 0:
476
+ continue
477
+ try:
478
+ pread(p, n)
479
+ except OSError:
480
+ probe(p, n)
481
+
482
+ pos, chunk = 0, 64 << 20
483
+ while pos < end:
484
+ want = min(chunk, end - pos)
485
+ try:
486
+ got = len(pread(pos, want))
487
+ except OSError:
488
+ probe(pos, want)
489
+ pos += want
490
+ got = None
491
+ if got is not None:
492
+ if got == 0: # outside the try: NtfsError subclasses OSError
493
+ raise NtfsError(0, f'device ends prematurely at byte {pos}')
494
+ pos += got
495
+ if progress:
496
+ progress(pos, end)
497
+ return sorted(set(bad))
498
+
499
+
500
+ def _set_run(bitmap: bytearray, lcn: int, length: int) -> None:
501
+ '''Mark clusters [lcn, lcn+length) used in an LSB-first bitmap.'''
502
+ end = lcn + length
503
+ first_full = (lcn + 7) // 8
504
+ last_full = end // 8
505
+ for c in range(lcn, min(first_full * 8, end)):
506
+ bitmap[c >> 3] |= 1 << (c & 7)
507
+ if last_full > first_full:
508
+ bitmap[first_full:last_full] = b'\xff' * (last_full - first_full)
509
+ for c in range(max(last_full * 8, lcn), end):
510
+ bitmap[c >> 3] |= 1 << (c & 7)
511
+
512
+
513
+ # ── volume handling ──────────────────────────────────────────────────────────
514
+
515
+ def _mount_table(mounts: str) -> list[tuple[str, str]]:
516
+ '''(source, mountpoint) pairs — /proc/mounts where it exists, the mount(8)
517
+ "SRC on MNT (opts)" output on macOS/FreeBSD.'''
518
+ if os.path.exists(mounts):
519
+ with open(mounts) as fh:
520
+ return [tuple(line.split()[:2]) for line in fh]
521
+ out = subprocess.run(['mount'], capture_output=True, text=True).stdout
522
+ pairs = []
523
+ for line in out.splitlines():
524
+ if ' on ' in line:
525
+ src, _, rest = line.partition(' on ')
526
+ mnt = rest.rsplit(' (', 1)[0].split(' type ')[0]
527
+ pairs.append((src.strip(), mnt.strip()))
528
+ return pairs
529
+
530
+
531
+ def assert_not_mounted(device: str, mounts: str = '/proc/mounts',
532
+ sys_block: str = '/sys/class/block') -> None:
533
+ real = os.path.realpath(device)
534
+ for src, mnt in _mount_table(mounts):
535
+ if not src.startswith('/'):
536
+ continue
537
+ src_real = os.path.realpath(src)
538
+ if src_real == real:
539
+ raise SystemExit(f'{device} is mounted ({mnt}) — unmount it first')
540
+ # An image file attached to a loop device mounts as /dev/loopN (or
541
+ # a partition /dev/loopNpM); the origin is only visible in the
542
+ # loop's sysfs backing_file. The ../ variant reaches the parent
543
+ # loop device from a partition's sysfs node.
544
+ name = os.path.basename(src_real)
545
+ for backing in (f'{sys_block}/{name}/loop/backing_file',
546
+ f'{sys_block}/{name}/../loop/backing_file'):
547
+ try:
548
+ with open(backing) as bf:
549
+ backing_real = os.path.realpath(bf.read().strip())
550
+ except OSError:
551
+ continue
552
+ if backing_real == real:
553
+ raise SystemExit(f'{device} is mounted via {src_real} on {mnt} '
554
+ '— detach/unmount it first')
555
+
556
+
557
+ class RawVolume:
558
+ '''Read-only NTFS access: boot-sector bootstrap, $MFT runlist (including
559
+ $ATTRIBUTE_LIST-spanning $DATA), raw attribute and $I30 index walking,
560
+ path resolution, and all the analysis passes (MFT survey, cluster audit,
561
+ $Secure and USN validation). Pure Python — no external library. Every
562
+ write refuses here; the RawVolumeRW subclass adds the repair engine.'''
563
+
564
+
565
+ def _invalidate_mft_cache(self) -> None:
566
+ '''Any repair may rewrite MFT records — drop the raw-read cache.'''
567
+ getattr(self, '_mft_chunk_cache', {}).clear()
568
+
569
+
570
+ def _load_record(self, rec_no: int) -> bytearray | None:
571
+ '''read_record + FILE magic + fixups — the standard triple. None when
572
+ the record is unreadable, not a FILE record, or torn.'''
573
+ raw = self.read_record(rec_no)
574
+ if raw is None or raw[:4] != b'FILE':
575
+ return None
576
+ rec = bytearray(raw)
577
+ return rec if _apply_fixups(rec) else None
578
+
579
+
580
+ def _mref_live(self, mref: int) -> bool:
581
+ '''True iff the referenced record is readable, in use, and still carries
582
+ the reference's sequence number (i.e. the reference is not stale).'''
583
+ rec = self._load_record(mref & MREF_MASK)
584
+ return (rec is not None
585
+ and bool(struct.unpack_from('<H', rec, 22)[0] & 1)
586
+ and struct.unpack_from('<H', rec, 16)[0] == mref >> 48)
587
+
588
+
589
+ def nr_clusters(self) -> int:
590
+ if getattr(self, '_nr_clusters', None) is None:
591
+ with open(self.device, 'rb') as fh:
592
+ boot = fh.read(72)
593
+ bps = struct.unpack_from('<H', boot, 11)[0]
594
+ spc = boot[13]
595
+ spc = 2 ** (256 - spc) if spc > 0x80 else spc
596
+ sectors = struct.unpack_from('<Q', boot, 40)[0]
597
+ self._cluster_size = bps * spc
598
+ self._nr_clusters = sectors // spc
599
+ return self._nr_clusters
600
+
601
+
602
+ def cluster_size(self) -> int:
603
+ self.nr_clusters()
604
+ return self._cluster_size
605
+
606
+
607
+ # -- chkdsk stage-5: rebuild the true cluster map, diff against $Bitmap --
608
+
609
+ def cluster_audit(self, survey: dict | None = None) -> dict:
610
+ '''Rebuild the true used-cluster map and diff it against $Bitmap.
611
+ missing > 0 means live data sits on clusters marked free (danger);
612
+ extra > 0 is leaked space. `failures` non-empty means the map is
613
+ incomplete and MUST NOT be written back.
614
+
615
+ The map comes from the raw runlist decode of a single $MFT
616
+ pass (mft_survey, reusable across stages).'''
617
+
618
+ nc = self.nr_clusters()
619
+ s = survey if survey and survey.get('used') is not None \
620
+ else self.mft_survey(want_used=True)
621
+ return self._bitmap_diff(nc, s['used'], list(s['map_failures']))
622
+
623
+
624
+ def _bitmap_diff(self, nc: int, used: bytearray, failures: list[str]) -> dict:
625
+ ondisk = self._read_whole_attr(FILE_BITMAP, AT_DATA)
626
+ n = len(used)
627
+ if len(ondisk) < n:
628
+ failures.append(f'$Bitmap shorter than expected ({len(ondisk)} < {n})')
629
+ ondisk = ondisk.ljust(n, b'\0')
630
+ mask = (1 << nc) - 1 # ignore padding bits past the last real cluster
631
+ u = int.from_bytes(bytes(used), 'little') & mask
632
+ d = int.from_bytes(ondisk[:n], 'little') & mask
633
+ extra, missing = d & ~u, u & ~d
634
+ return {'nr_clusters': nc, 'used_count': u.bit_count(),
635
+ 'bitmap_count': d.bit_count(),
636
+ 'extra': extra.bit_count(), 'missing': missing.bit_count(),
637
+ 'used_bytes': bytes(used), 'ondisk_tail': ondisk[n - 1] if n else 0,
638
+ 'failures': failures}
639
+
640
+
641
+ # -- chkdsk stage 3: $Secure ($SDS stream + $SII/$SDH indexes) --
642
+
643
+ def secure_check(self, survey: dict | None = None) -> dict:
644
+ '''Validate the security descriptor store: every $SDS entry's header,
645
+ hash and 256 KiB mirror; the $SII (id) and $SDH (hash) indexes agree
646
+ with $SDS entry-for-entry; and every in-use file's security_id
647
+ references an existing descriptor.'''
648
+
649
+ info = {'present': False, 'problems': [], 'notes': [], 'mirror_fixes': [],
650
+ 'index_problems': [], 'ref_missing': [], 'sds_entries': {},
651
+ 'descriptors': 0, 'files_checked': 0}
652
+ try:
653
+ sds = self._read_whole_attr(FILE_SECURE, AT_DATA, SDS_NAME, 4)
654
+ except NtfsError:
655
+ info['notes'].append('no $Secure/$SDS — pre-NTFS-3.0 style volume')
656
+ return info
657
+ info['present'] = True
658
+
659
+ entries, sds_problems, fixes = _parse_sds(sds)
660
+ info['sds_entries'] = entries
661
+ info['descriptors'] = len(entries)
662
+ info['problems'].extend(sds_problems)
663
+ info['mirror_fixes'] = fixes
664
+
665
+ for label, name, key_fmt in (('$SII', SII_NAME, '<I'), ('$SDH', SDH_NAME, '<II')):
666
+ try:
667
+ root = self._read_whole_attr(FILE_SECURE, AT_INDEX_ROOT, name, 4)
668
+ except NtfsError:
669
+ info['index_problems'].append(f'{label}: $INDEX_ROOT missing')
670
+ continue
671
+ alloc = bmp = None
672
+ try:
673
+ alloc = self._read_whole_attr(FILE_SECURE, AT_INDEX_ALLOCATION, name, 4)
674
+ bmp = self._read_whole_attr(FILE_SECURE, AT_BITMAP, name, 4)
675
+ except NtfsError:
676
+ pass # small index — resident root only
677
+ key_fn = (lambda k, f=key_fmt: struct.unpack_from(f, k)
678
+ if len(k) >= struct.calcsize(f) else ())
679
+ idx, idx_problems = _view_index_entries(root, alloc, bmp, label,
680
+ key_fn=key_fn)
681
+ info['index_problems'].extend(idx_problems)
682
+ seen = set()
683
+ for key, data in idx:
684
+ if len(data) < 20:
685
+ info['index_problems'].append(f'{label}: entry data too short')
686
+ continue
687
+ h, sid, off, length = struct.unpack_from('<IIQI', data, 0)
688
+ if label == '$SII':
689
+ kid = struct.unpack_from('<I', key, 0)[0] if len(key) >= 4 else -1
690
+ ok = kid == sid
691
+ else:
692
+ kh, kid = (struct.unpack_from('<II', key, 0)
693
+ if len(key) >= 8 else (-1, -1))
694
+ ok = kid == sid and kh == h
695
+ if not ok:
696
+ info['index_problems'].append(f'{label}: key does not match its '
697
+ f'entry data (id {sid})')
698
+ continue
699
+ seen.add(sid)
700
+ if entries.get(sid) != (h, off, length):
701
+ info['index_problems'].append(f'{label}: entry for id {sid} '
702
+ 'disagrees with $SDS')
703
+ for sid in entries.keys() - seen:
704
+ info['index_problems'].append(f'{label}: id {sid} missing from index')
705
+
706
+ # every in-use file's security_id must exist in $SDS (id 0 = none) —
707
+ # the referenced-id map comes from the shared $MFT survey
708
+ s = survey if survey and survey.get('sec_refs') is not None \
709
+ else self.mft_survey(want_security=True)
710
+ info['files_checked'] = sum(len(r) for r in s['sec_refs'].values())
711
+ for sid, records in sorted(s['sec_refs'].items()):
712
+ if sid not in entries:
713
+ info['ref_missing'].append(
714
+ f'security id {sid} referenced by {len(records)} file(s) '
715
+ f'(e.g. record {records[0]}) but absent from $SDS')
716
+ return info
717
+
718
+
719
+ # -- $Extend view indexes: $Reparse ($R) and $ObjId ($O) vs the records --
720
+
721
+ def view_index_check(self, survey: dict | None = None) -> dict:
722
+ '''Walk the $Reparse and $ObjId view indexes (key-ordered) and check
723
+ both directions: every entry points at a live record carrying the
724
+ matching attribute, and every record carrying it has an entry.'''
725
+ out = {'problems': [], 'notes': [], 'checked': 0}
726
+ s = survey or self.mft_survey()
727
+ for path, name_const, attr_want, kind, recs in (
728
+ ('/$Extend/$Reparse', R_NAME, 0xC0, '$Reparse', s['reparse_recs']),
729
+ ('/$Extend/$ObjId', O_NAME, 0x40, '$ObjId', s['objid_recs'])):
730
+ try:
731
+ no = self.resolve(path)
732
+ root = self._read_whole_attr(no, AT_INDEX_ROOT, name_const, 2)
733
+ except NtfsError:
734
+ out['notes'].append(f'{kind}: absent')
735
+ if recs:
736
+ out['problems'].append(f'{kind}: index absent but {len(recs)} '
737
+ 'record(s) carry the attribute')
738
+ continue
739
+ alloc = bmp = None
740
+ try:
741
+ alloc = self._read_whole_attr(no, AT_INDEX_ALLOCATION, name_const, 2)
742
+ bmp = self._read_whole_attr(no, AT_BITMAP, name_const, 2)
743
+ except NtfsError:
744
+ pass
745
+ key_fn = (lambda k: struct.unpack_from(f'<{len(k) // 4}I', k)
746
+ if len(k) >= 4 else ())
747
+ entries, probs = _view_index_entries(root, alloc, bmp, kind, key_fn=key_fn)
748
+ out['problems'] += probs
749
+ seen: set[int] = set()
750
+ for key, data in entries:
751
+ out['checked'] += 1
752
+ if kind == '$Reparse' and len(key) >= 12:
753
+ mref = struct.unpack_from('<Q', key, 4)[0]
754
+ elif kind == '$ObjId' and len(data) >= 8:
755
+ mref = struct.unpack_from('<Q', data, 0)[0]
756
+ else:
757
+ out['problems'].append(f'{kind}: malformed entry')
758
+ continue
759
+ m_no = mref & MREF_MASK
760
+ r = self._load_record(m_no)
761
+ ok = (r is not None
762
+ and struct.unpack_from('<H', r, 22)[0] & 1
763
+ and struct.unpack_from('<H', r, 16)[0] == mref >> 48
764
+ and _has_attr(r, attr_want))
765
+ if ok:
766
+ seen.add(m_no)
767
+ else:
768
+ out['problems'].append(f'{kind}: entry points at record {m_no} '
769
+ 'which is dead, reused or lacks the attribute')
770
+ for m_no in sorted(recs - seen)[:8]:
771
+ out['problems'].append(f'{kind}: record {m_no} carries the attribute '
772
+ 'but has no index entry')
773
+ return out
774
+
775
+
776
+ # -- lost files: in-use records no index entry references (chkdsk's
777
+ # "recovering orphaned file") --
778
+
779
+ def find_lost_files(self, referenced: set[int]) -> list[dict]:
780
+ '''Every in-use base record (past the system range) carrying $FILE_NAME
781
+ attributes but absent from `referenced` — the set of records that
782
+ some valid index entry points at. parent_ok means the claimed parent
783
+ is a live directory with a matching sequence, i.e. reconnectable.'''
784
+ lost: list[dict] = []
785
+ rec_size = self.mft_record_size()
786
+ chunk_size, offset = 1 << 20, 0
787
+ while True:
788
+ chunk = self._mft_bulk(offset, chunk_size)
789
+ usable = len(chunk) - (len(chunk) % rec_size)
790
+ if not usable:
791
+ break
792
+ chunk = chunk[:usable]
793
+ for rec_off in range(0, usable, rec_size):
794
+ rec_no = (offset + rec_off) // rec_size
795
+ if rec_no < 24 or rec_no in referenced:
796
+ continue # system range is never index-reconnected
797
+ raw = chunk[rec_off:rec_off + rec_size]
798
+ if raw[:4] != b'FILE':
799
+ continue
800
+ rec = bytearray(raw)
801
+ if not _apply_fixups(rec):
802
+ continue
803
+ flags = struct.unpack_from('<H', rec, 22)[0]
804
+ base = struct.unpack_from('<Q', rec, 32)[0]
805
+ if not flags & 1 or base & MREF_MASK:
806
+ continue
807
+ frozen = bytes(rec)
808
+ names = list(_file_name_attrs(frozen))
809
+ if not names:
810
+ continue
811
+ seq = struct.unpack_from('<H', frozen, 16)[0]
812
+ parent = struct.unpack_from('<Q', names[0], 0)[0]
813
+ p_no, p_seq = parent & MREF_MASK, parent >> 48
814
+ # reconnectable = the claimed parent is a live DIRECTORY whose
815
+ # sequence still matches (stricter than _mref_live)
816
+ p_rec = self._load_record(p_no)
817
+ parent_ok = False
818
+ if p_rec is not None:
819
+ p_flags = struct.unpack_from('<H', p_rec, 22)[0]
820
+ parent_ok = bool(p_flags & 1 and p_flags & 2
821
+ and struct.unpack_from('<H', p_rec, 16)[0] == p_seq)
822
+ lost.append({
823
+ 'record': rec_no, 'seq': seq, 'parent': p_no,
824
+ 'parent_ok': parent_ok, 'fn_list': [bytes(fn) for fn in names],
825
+ 'name': names[0][66:66 + 2 * names[0][64]].decode(
826
+ 'utf-16-le', 'replace')})
827
+ offset += usable
828
+ return lost
829
+
830
+
831
+ # -- chkdsk stage 4 support: attribute bad clusters to their owners --
832
+
833
+ def surface_owners(self, bad: set[int]) -> list[dict]:
834
+ '''Attribute unreadable clusters to their owning records by decoding
835
+ every in-use record's runlists raw (one bulk $MFT pass); whatever
836
+ no record claims is reported as free space.'''
837
+ order = sorted(bad)
838
+ owners: list[dict] = []
839
+ claimed: set[int] = set()
840
+ nc = self.nr_clusters()
841
+ rec_size = self.mft_record_size()
842
+ chunk_size, offset = 1 << 20, 0
843
+ while True:
844
+ chunk = self._mft_bulk(offset, chunk_size)
845
+ usable = len(chunk) - (len(chunk) % rec_size)
846
+ if not usable:
847
+ break
848
+ chunk = chunk[:usable]
849
+ for rec_off in range(0, usable, rec_size):
850
+ raw = chunk[rec_off:rec_off + rec_size]
851
+ if raw[:4] != b'FILE':
852
+ continue
853
+ rec = bytearray(raw)
854
+ if not _apply_fixups(rec):
855
+ continue
856
+ if not struct.unpack_from('<H', rec, 22)[0] & 1:
857
+ continue
858
+ frozen = bytes(rec)
859
+ rec_no = (offset + rec_off) // rec_size
860
+ a_off = struct.unpack_from('<H', frozen, 20)[0]
861
+ while a_off + 24 <= len(frozen):
862
+ attr_type = struct.unpack_from('<I', frozen, a_off)[0]
863
+ if attr_type == 0xFFFFFFFF:
864
+ break
865
+ length = struct.unpack_from('<I', frozen, a_off + 4)[0]
866
+ if length < 24 or a_off + length > len(frozen):
867
+ break
868
+ if frozen[a_off + 8] == 1:
869
+ _p, _ph, runs = _check_mapping_pairs(frozen, a_off, length, nc)
870
+ for lcn, run_len, _vcn in runs:
871
+ lo = bisect.bisect_left(order, lcn)
872
+ hi = bisect.bisect_right(order, lcn + run_len - 1)
873
+ if hi > lo:
874
+ hits = order[lo:hi]
875
+ names = ', '.join(
876
+ repr(fn[66:66 + 2 * fn[64]].decode('utf-16-le',
877
+ 'replace'))
878
+ for fn in _file_name_attrs(frozen)) or '(no name)'
879
+ owners.append({'record': rec_no, 'attr': attr_type,
880
+ 'names': names, 'clusters': hits})
881
+ claimed.update(hits)
882
+ a_off += length
883
+ offset += usable
884
+ free = sorted(bad - claimed)
885
+ if free:
886
+ owners.append({'record': None, 'attr': None,
887
+ 'names': '(free space)', 'clusters': free})
888
+ return owners
889
+
890
+
891
+ def __enter__(self):
892
+ return self
893
+
894
+
895
+ def __exit__(self, *_exc):
896
+ self.close()
897
+
898
+
899
+ # -- NTFS name equality: this volume's $UpCase table, not Python casing --
900
+
901
+ def _upcase_table(self) -> array.array | None:
902
+ if self._upcase is None:
903
+ self._upcase = False # sticky: don't retry a failed load
904
+ try:
905
+ raw = self._read_whole_attr(FILE_UPCASE, AT_DATA)
906
+ except NtfsError:
907
+ raw = b''
908
+ if len(raw) >= 512:
909
+ table = array.array('H', raw[:len(raw) & ~1])
910
+ if sys.byteorder == 'big':
911
+ table.byteswap()
912
+ self._upcase = table
913
+ return self._upcase or None
914
+
915
+
916
+ def names_equal(self, a: str, b: str) -> bool:
917
+ '''Case-insensitive equality the way THIS volume defines it ($UpCase);
918
+ degrades to Python casing only if $UpCase cannot be read.'''
919
+ table = self._upcase_table()
920
+ if table is None:
921
+ return a.lower() == b.lower()
922
+ ea, eb = a.encode('utf-16-le'), b.encode('utf-16-le')
923
+ if len(ea) != len(eb):
924
+ return False
925
+ n = len(table)
926
+ return all((table[x] if x < n else x) == (table[y] if y < n else y)
927
+ for x, y in zip(struct.unpack(f'<{len(ea) // 2}H', ea),
928
+ struct.unpack(f'<{len(eb) // 2}H', eb)))
929
+
930
+
931
+ # -- the unified $MFT sweep: one bulk pass serves stages 1, 3 and 5 --
932
+ def mft_survey(self, want_used: bool = False, want_security: bool = False) -> dict:
933
+ '''One pass over $MFT computing stage-1 health (record counts, torn
934
+ records, attribute-chain and runlist problems — entries with a
935
+ 'fix_mp_off' are safely auto-repairable torn truncates), and
936
+ optionally the stage-5 used-cluster map from the decoded runlists
937
+ and the stage-3 map of referenced security ids. Reading $MFT once
938
+ instead of once per stage (and decoding runlists raw instead of
939
+ 729k per-inode round-trips) is the main /f speedup.'''
940
+ nc = self.nr_clusters()
941
+ rec_size = self.mft_record_size()
942
+ total = in_use = dirs = torn = 0
943
+ problems: list[dict] = []
944
+ used = bytearray((nc + 7) // 8) if want_used else None
945
+ # per-RECORD in-use bits (for the $MFT $BITMAP reconciliation) — not
946
+ # to be confused with `used`, which is the per-CLUSTER map for stage 5
947
+ n_slots = self._mft_size // rec_size
948
+ rec_used = bytearray((n_slots + 7) // 8) if want_used else None
949
+ map_failures: list[str] = []
950
+ sec_refs: dict[int, list[int]] = {}
951
+ reparse_recs: set[int] = set()
952
+ objid_recs: set[int] = set()
953
+ chunk_size, offset = 1 << 20, 0
954
+ while True:
955
+ chunk = self._mft_bulk(offset, chunk_size)
956
+ usable = len(chunk) - (len(chunk) % rec_size)
957
+ if not usable:
958
+ break
959
+ chunk = chunk[:usable]
960
+ for rec_off in range(0, usable, rec_size):
961
+ raw = chunk[rec_off:rec_off + rec_size]
962
+ if raw[:4] != b'FILE':
963
+ continue
964
+ total += 1
965
+ rec_no = (offset + rec_off) // rec_size
966
+ rec = bytearray(raw)
967
+ if not _apply_fixups(rec):
968
+ torn += 1
969
+ # flags live in the first sector — readable even when torn
970
+ if struct.unpack_from('<H', raw, 22)[0] & 1 and want_used:
971
+ map_failures.append(f'record {rec_no}: torn but in use — '
972
+ 'its clusters are unknowable')
973
+ # keep its bitmap bit set: clearing a slot that still
974
+ # claims in-use would invite reuse under a live record
975
+ rec_used[rec_no >> 3] |= 1 << (rec_no & 7)
976
+ continue
977
+ flags = struct.unpack_from('<H', rec, 22)[0]
978
+ in_use += flags & 1
979
+ dirs += bool(flags & 1) and bool(flags & 2)
980
+ if flags & 1 and want_used:
981
+ rec_used[rec_no >> 3] |= 1 << (rec_no & 7)
982
+ if not flags & 1:
983
+ continue
984
+ frozen = bytes(rec)
985
+ base = struct.unpack_from('<Q', frozen, 32)[0]
986
+ if base & MREF_MASK and not self._mref_live(base):
987
+ # extension record whose base is dead or reused
988
+ problems.append({'record': rec_no, 'attr_type': 0,
989
+ 'fix_mp_off': None, 'frozen': frozen,
990
+ 'problem': f'orphaned extension record: base '
991
+ f'{base & MREF_MASK} is dead or reused'})
992
+ fn_count, has_attr_list = 0, False
993
+ a_off = struct.unpack_from('<H', frozen, 20)[0]
994
+ while a_off + 8 <= len(frozen):
995
+ attr_type = struct.unpack_from('<I', frozen, a_off)[0]
996
+ if attr_type == 0xFFFFFFFF:
997
+ break
998
+ length = struct.unpack_from('<I', frozen, a_off + 4)[0]
999
+ if length < 24 or a_off + length > len(frozen):
1000
+ problems.append({'record': rec_no, 'attr_type': attr_type,
1001
+ 'fix_mp_off': None, 'frozen': frozen,
1002
+ 'problem': f'broken attribute chain at +{a_off}'})
1003
+ if want_used:
1004
+ map_failures.append(f'record {rec_no}: broken attribute chain')
1005
+ break
1006
+ if attr_type == AT_FILE_NAME and frozen[a_off + 8] == 0:
1007
+ fn_count += 1
1008
+ elif attr_type == 0xC0:
1009
+ reparse_recs.add(rec_no)
1010
+ elif attr_type == 0x40:
1011
+ objid_recs.add(rec_no)
1012
+ elif attr_type == AT_ATTRIBUTE_LIST and frozen[a_off + 8] == 0:
1013
+ has_attr_list = True
1014
+ v_len = struct.unpack_from('<I', frozen, a_off + 16)[0]
1015
+ v_ofs = struct.unpack_from('<H', frozen, a_off + 20)[0]
1016
+ listing = frozen[a_off + v_ofs:a_off + v_ofs + v_len]
1017
+ pos = 0
1018
+ while pos + 26 <= len(listing):
1019
+ e_len = struct.unpack_from('<H', listing, pos + 4)[0]
1020
+ if e_len < 26:
1021
+ problems.append({'record': rec_no, 'attr_type': 0x20,
1022
+ 'fix_mp_off': None, 'frozen': frozen,
1023
+ 'problem': 'broken $ATTRIBUTE_LIST entry'})
1024
+ break
1025
+ lref = struct.unpack_from('<Q', listing, pos + 16)[0]
1026
+ l_no = lref & MREF_MASK
1027
+ if l_no != rec_no and not self._mref_live(lref):
1028
+ problems.append(
1029
+ {'record': rec_no, 'attr_type': 0x20,
1030
+ 'fix_mp_off': None, 'frozen': frozen,
1031
+ 'problem': '$ATTRIBUTE_LIST points at dead/'
1032
+ f'reused extension record {l_no}'})
1033
+ pos += e_len
1034
+ elif attr_type == AT_ATTRIBUTE_LIST:
1035
+ has_attr_list = True # non-resident listing: skip content
1036
+ if (want_security and attr_type == AT_STANDARD_INFORMATION
1037
+ and frozen[a_off + 8] == 0):
1038
+ value_len = struct.unpack_from('<I', frozen, a_off + 16)[0]
1039
+ value_ofs = struct.unpack_from('<H', frozen, a_off + 20)[0]
1040
+ if value_len >= 72:
1041
+ sid = struct.unpack_from('<I', frozen,
1042
+ a_off + value_ofs + 52)[0]
1043
+ if sid:
1044
+ sec_refs.setdefault(sid, []).append(rec_no)
1045
+ if frozen[a_off + 8] == 1: # non-resident
1046
+ problem, phantom, runs = _check_mapping_pairs(
1047
+ frozen, a_off, length, nc)
1048
+ if problem:
1049
+ fix = None
1050
+ if phantom:
1051
+ sizes = struct.unpack_from('<QQQ', frozen, a_off + 40)
1052
+ mp_abs = a_off + struct.unpack_from(
1053
+ '<H', frozen, a_off + 32)[0]
1054
+ if sizes == (0, 0, 0) and mp_abs % 512 < 510:
1055
+ fix = mp_abs
1056
+ problems.append({'record': rec_no, 'attr_type': attr_type,
1057
+ 'problem': problem, 'fix_mp_off': fix,
1058
+ 'frozen': frozen})
1059
+ # phantom runs are never live data (header says empty)
1060
+ # so they don't make the used map incomplete
1061
+ if want_used and not phantom:
1062
+ map_failures.append(
1063
+ f'record {rec_no} attr {attr_type:#x}: {problem}')
1064
+ elif want_used:
1065
+ for lcn, run_len, _vcn in runs:
1066
+ _set_run(used, lcn, run_len)
1067
+ a_off += length
1068
+ # hard-link count vs the record's own $FILE_NAME attrs (spilled
1069
+ # attr lists skipped: names may live in extension records)
1070
+ if not has_attr_list and fn_count and not base & MREF_MASK:
1071
+ link_count = struct.unpack_from('<H', frozen, 18)[0]
1072
+ if link_count != fn_count:
1073
+ problems.append({'record': rec_no, 'attr_type': AT_FILE_NAME,
1074
+ 'fix_mp_off': None, 'frozen': frozen,
1075
+ 'problem': f'hard-link count {link_count} but '
1076
+ f'{fn_count} $FILE_NAME attribute(s)'})
1077
+ offset += usable
1078
+ for prob in problems:
1079
+ prob['names'] = ', '.join(
1080
+ repr(fn[66:66 + 2 * fn[64]].decode('utf-16-le', 'replace'))
1081
+ for fn in _file_name_attrs(prob.pop('frozen'))) or '(no name)'
1082
+ return {'total': total, 'in_use': in_use, 'dirs': dirs, 'torn': torn,
1083
+ 'runlist_problems': problems, 'used': used,
1084
+ 'rec_used': rec_used, 'n_slots': n_slots,
1085
+ 'map_failures': map_failures, 'sec_refs': sec_refs,
1086
+ 'reparse_recs': reparse_recs, 'objid_recs': objid_recs}
1087
+
1088
+
1089
+ def _entry_status(self, dir_mft_no: int, entry: dict) -> str:
1090
+ '''Validate one index entry against the raw MFT record — chkdsk stage-2
1091
+ semantics. Deliberately NOT ntfs_inode_open(mref): a cached inode hit
1092
+ skips the sequence check, so a stale entry can scan "ok" whenever the
1093
+ record's current owner was opened earlier on this volume handle.'''
1094
+
1095
+ raw = self.read_record(entry['record'])
1096
+ if raw is None or raw[:4] != b'FILE':
1097
+ return 'DANGLING (record unreadable)'
1098
+ rec = bytearray(raw)
1099
+ if not _apply_fixups(rec):
1100
+ return 'DANGLING (torn record)'
1101
+ flags = struct.unpack_from('<H', rec, 22)[0]
1102
+ entry['is_dir'] = bool(flags & 2)
1103
+ if not flags & 1:
1104
+ return 'DANGLING (record not in use)'
1105
+ rec_seq = struct.unpack_from('<H', rec, 16)[0]
1106
+ if rec_seq != entry['seq']:
1107
+ return f'DANGLING (seq mismatch: dirent {entry["seq"]} vs record {rec_seq})'
1108
+ frozen = bytes(rec)
1109
+ for fn in _file_name_attrs(frozen):
1110
+ parent = struct.unpack_from('<Q', fn, 0)[0]
1111
+ fn_name = fn[66:66 + 2 * fn[64]].decode('utf-16-le', 'replace')
1112
+ if parent & MREF_MASK == dir_mft_no and self.names_equal(fn_name, entry['name']):
1113
+ return 'ok'
1114
+ # $FILE_NAME may live in an extension record when an $ATTRIBUTE_LIST
1115
+ # exists — fall back to the full attr-list enumeration for those.
1116
+ if _has_attr(frozen, AT_ATTRIBUTE_LIST):
1117
+ info = self.record_info(entry['record'])
1118
+ if any(self.names_equal(n['name'], entry['name'])
1119
+ and n['parent_record'] == dir_mft_no
1120
+ for n in info.get('names', [])):
1121
+ return 'ok'
1122
+ return 'DANGLING (cross-linked: record does not carry this name)'
1123
+
1124
+
1125
+ def _classify_found(self, dir_mft_no: int, name: str, found: dict) -> dict:
1126
+ result = {'name': name, 'dir_record': dir_mft_no,
1127
+ 'dirent_record': found['record'], 'dirent_seq': found['seq']}
1128
+
1129
+ # Healthy = the record header matches the dirent's (record, seq) AND the
1130
+ # record carries this (name, parent) in a $FILE_NAME — chkdsk validates
1131
+ # both. Deliberately NOT ntfs_inode_open(mref): a prior seq-bypass open
1132
+ # (ours, below) leaves the inode cached and a cached hit skips the seq
1133
+ # check, so that test would go stale after the first classify.
1134
+ record = self.record_info(found['record'])
1135
+ result['record_info'] = record
1136
+ names_match = record.get('open') and any(
1137
+ self.names_equal(entry['name'], name)
1138
+ and entry['parent_record'] == dir_mft_no for entry in record['names'])
1139
+ if record['open'] and record.get('in_use') and record['seq'] == found['seq']:
1140
+ if names_match:
1141
+ result.update(state='healthy', action='nothing to do')
1142
+ else:
1143
+ result.update(
1144
+ state='cross-linked',
1145
+ action='rm ONLY — the dirent points into a live record that does '
1146
+ 'not carry this name; the real file lives elsewhere')
1147
+ elif not record['open'] or not record.get('in_use'):
1148
+ result.update(
1149
+ state='record-free',
1150
+ action='rm — the entry is the only leftover; nothing else to clean')
1151
+ elif names_match:
1152
+ result.update(
1153
+ state='orphan',
1154
+ action='purge — record still belongs to this name; one call frees '
1155
+ 'dirent, record and clusters')
1156
+ else:
1157
+ names = ', '.join(repr(entry['name']) for entry in record['names']) or '(none)'
1158
+ result.update(
1159
+ state='record-reused',
1160
+ action=f'rm ONLY — the record now belongs to a live file ({names}); '
1161
+ 'do not touch it')
1162
+ return result
1163
+
1164
+
1165
+ # -- chkdsk-style index rebuild: recover a directory from the MFT side --
1166
+
1167
+ def probe_dir_no(self, mft_no: int) -> dict:
1168
+ try:
1169
+ entries = [e for e in self.scan_dir_no(mft_no) if e['status'] != 'dir-self']
1170
+ return {'readable': True, 'entries': entries}
1171
+ except NtfsError as exc:
1172
+ return {'readable': False, 'error': str(exc)}
1173
+
1174
+
1175
+ def probe_dir(self, path: str) -> dict:
1176
+ '''Try to walk the index; a torn INDX block surfaces as a readdir error '''
1177
+
1178
+ try:
1179
+ return self.probe_dir_no(self._resolve(path))
1180
+ except NtfsError as exc:
1181
+ return {'readable': False, 'error': str(exc)}
1182
+
1183
+
1184
+ def children_from_mft(self, dir_mft_no: int) -> dict:
1185
+ '''Every in-use base MFT record whose $FILE_NAME claims dir_mft_no as
1186
+ parent — the same redundancy chkdsk rebuilds a torn index from '''
1187
+
1188
+ children: list[dict] = []
1189
+ unreadable = 0
1190
+ dir_seq = None
1191
+ rec_size = self.mft_record_size()
1192
+ chunk_size, offset = 1 << 20, 0
1193
+ while True:
1194
+ chunk = self._mft_bulk(offset, chunk_size)
1195
+ # Only advance by whole records: a short read must not shift
1196
+ # every subsequent record off its alignment.
1197
+ usable = len(chunk) - (len(chunk) % rec_size)
1198
+ if not usable:
1199
+ break # tail shorter than one record
1200
+ chunk = chunk[:usable]
1201
+ for rec_off in range(0, usable, rec_size):
1202
+ raw = chunk[rec_off:rec_off + rec_size]
1203
+ if raw[:4] != b'FILE':
1204
+ continue
1205
+ rec_no = (offset + rec_off) // rec_size
1206
+ rec = bytearray(raw)
1207
+ if not _apply_fixups(rec):
1208
+ unreadable += 1
1209
+ continue
1210
+ flags = struct.unpack_from('<H', rec, 22)[0]
1211
+ base = struct.unpack_from('<Q', rec, 32)[0]
1212
+ seq = struct.unpack_from('<H', rec, 16)[0]
1213
+ if rec_no == dir_mft_no:
1214
+ dir_seq = seq
1215
+ if not flags & 1 or base & MREF_MASK: # free, or extension record
1216
+ continue
1217
+ for fn in _file_name_attrs(rec):
1218
+ parent = struct.unpack_from('<Q', fn, 0)[0]
1219
+ if parent & MREF_MASK != dir_mft_no or rec_no == dir_mft_no:
1220
+ continue
1221
+ n_len, n_type = fn[64], fn[65]
1222
+ children.append({
1223
+ 'record': rec_no, 'seq': seq, 'type': n_type,
1224
+ 'parent_seq': parent >> 48,
1225
+ 'name': fn[66:66 + 2 * n_len].decode('utf-16-le', 'replace'),
1226
+ 'fn_bytes': bytes(fn),
1227
+ })
1228
+ offset += usable
1229
+ if dir_seq is None:
1230
+ raise NtfsError(0, f'directory record {dir_mft_no} not found in the $MFT scan')
1231
+ # A name whose parent reference carries a stale sequence belongs to a
1232
+ # previous incarnation of this directory — chkdsk would not resurrect
1233
+ # it into the rebuilt index, and neither do we.
1234
+ live = [c for c in children if c['parent_seq'] == dir_seq]
1235
+ return {'children': live, 'dir_seq': dir_seq,
1236
+ 'stale_parent': len(children) - len(live),
1237
+ 'unreadable_records': unreadable}
1238
+
1239
+
1240
+ def __init__(self, device: str, readonly: bool = True, base: int = 0):
1241
+ if not readonly:
1242
+ raise SystemExit('RawVolume is read-only — use RawVolumeRW for repairs')
1243
+ self.device = device
1244
+ self._upcase = None
1245
+ self._base = base # byte offset of the volume within the device
1246
+ self._fd = os.open(device, os.O_RDONLY | _O_BINARY)
1247
+ boot = _pread(self._fd, 512, self._base)
1248
+ if boot[3:7] != b'NTFS':
1249
+ raise NtfsError(0, f'{device}: no NTFS boot sector')
1250
+ bps = struct.unpack_from('<H', boot, 11)[0]
1251
+ spc = boot[13]
1252
+ spc = 2 ** (256 - spc) if spc > 0x80 else spc
1253
+ self._cluster_size = bps * spc
1254
+ self._nr_clusters = struct.unpack_from('<Q', boot, 40)[0] // spc
1255
+ mft_lcn = struct.unpack_from('<Q', boot, 48)[0]
1256
+ raw = struct.unpack_from('<b', boot, 64)[0]
1257
+ self._rec_size = 2 ** -raw if raw < 0 else raw * self._cluster_size
1258
+ if self._rec_size not in (1024, 2048, 4096):
1259
+ raise NtfsError(0, f'implausible MFT record size {self._rec_size}')
1260
+ # bootstrap caches + provisional $MFT map (just its first clusters,
1261
+ # enough to read record 0 and discover the real runlist)
1262
+ import collections
1263
+ self._blk_cache = collections.OrderedDict()
1264
+ self._attr_memo = {}
1265
+ self._mft_runs = [(mft_lcn, 4, 0)]
1266
+ self._mft_size = 4 * self._cluster_size
1267
+ self._mft_starts = [0]
1268
+ rec0 = self.read_record(0)
1269
+ if rec0 is None or rec0[:4] != b'FILE':
1270
+ raise NtfsError(0, 'cannot read $MFT record 0')
1271
+ self._mft_runs = self._stream_runs(0, AT_DATA, None)[1]
1272
+ self._mft_size = sum(r[1] for r in self._mft_runs) * self._cluster_size
1273
+ self._mft_starts = [r[2] * self._cluster_size for r in self._mft_runs]
1274
+ self._blk_cache.clear() # drop blocks read via the provisional map
1275
+ self._attr_memo.clear()
1276
+ try:
1277
+ vi = self._attr_value(3, 0x70, None)
1278
+ if vi and len(vi) >= 12 and struct.unpack_from('<H', vi, 10)[0] & 1:
1279
+ print(f'WARNING: {device} is marked dirty (unreplayed $LogFile) '
1280
+ '— read results may be slightly stale', file=sys.stderr)
1281
+ except NtfsError:
1282
+ pass
1283
+
1284
+ def close(self) -> None:
1285
+ if getattr(self, '_fd', None) is not None:
1286
+ os.close(self._fd)
1287
+ self._fd = None
1288
+
1289
+ # -- dirty flag: $VOLUME_INFORMATION (0x70) of $Volume (record 3) --
1290
+
1291
+ def _volume_info(self):
1292
+ rec = self._load_record(3)
1293
+ if rec is None:
1294
+ raise NtfsError(errno.EIO, '$Volume record unreadable')
1295
+ for a, t, _ln in _attrs(rec):
1296
+ if t == 0x70 and rec[a + 8] == 0:
1297
+ v_ofs = struct.unpack_from('<H', rec, a + 20)[0]
1298
+ return rec, a + v_ofs + 10 # u16 flags inside the value
1299
+ raise NtfsError(errno.ENOENT, '$VOLUME_INFORMATION not found')
1300
+
1301
+ def _dirty_flag(self) -> bool:
1302
+ rec, off = self._volume_info()
1303
+ return bool(struct.unpack_from('<H', rec, off)[0] & 1)
1304
+
1305
+ def volume_label(self) -> str:
1306
+ '''$VOLUME_NAME (0x60) of $Volume — UTF-16LE, may be absent (no label).'''
1307
+ rec = self._load_record(3)
1308
+ if rec is None:
1309
+ return ''
1310
+ for a, t, _ln in _attrs(rec):
1311
+ if t == 0x60 and rec[a + 8] == 0:
1312
+ vlen = struct.unpack_from('<I', rec, a + 16)[0]
1313
+ vofs = struct.unpack_from('<H', rec, a + 20)[0]
1314
+ return bytes(rec[a + vofs:a + vofs + vlen]).decode('utf-16-le',
1315
+ 'replace')
1316
+ return ''
1317
+
1318
+ # -- primitives --
1319
+
1320
+ def _dev_read(self, offset: int, size: int) -> bytes:
1321
+ return _pread(self._fd, size, self._base + offset)
1322
+
1323
+ def _runs_read(self, runs, offset: int, size: int) -> bytes:
1324
+ '''Read [offset, offset+size) of a stream laid out by vcn-ordered runs;
1325
+ holes (vcns no run covers) read as zeros. One pread per overlapping
1326
+ run, bisected start, zero-copy when one run covers the whole range.'''
1327
+ csz = self._cluster_size
1328
+ end = offset + size
1329
+ first = 0
1330
+ if len(runs) > 8: # skip straight to the first candidate run
1331
+ starts = (self._mft_starts if runs is self._mft_runs
1332
+ else [r[2] * csz for r in runs])
1333
+ first = max(bisect.bisect_right(starts, offset) - 1, 0)
1334
+ out = None
1335
+ for lcn, run_len, vcn in runs[first:]:
1336
+ r_start = vcn * csz
1337
+ if r_start >= end:
1338
+ break # runs are vcn-sorted: nothing further can overlap
1339
+ r_end = r_start + run_len * csz
1340
+ lo, hi = max(offset, r_start), min(end, r_end)
1341
+ if lo >= hi:
1342
+ continue
1343
+ data = _pread(self._fd, hi - lo, self._base + lcn * csz + (lo - r_start))
1344
+ if lo == offset and hi == end:
1345
+ return data # common case: whole range inside one run
1346
+ if out is None:
1347
+ out = bytearray(size)
1348
+ out[lo - offset:lo - offset + len(data)] = data
1349
+ return bytes(out) if out is not None else b'\x00' * size
1350
+
1351
+ def mft_record_size(self) -> int:
1352
+ return self._rec_size
1353
+
1354
+ BLOCK = 1 << 16 # record-read granularity: 64 KiB, LRU-cached
1355
+
1356
+ def _mft_bulk(self, offset: int, size: int) -> bytes:
1357
+ if offset >= self._mft_size:
1358
+ return b''
1359
+ return self._runs_read(self._mft_runs, offset,
1360
+ min(size, self._mft_size - offset))
1361
+
1362
+ def read_record(self, rec_no: int) -> bytes | None:
1363
+ '''64 KiB block-cached record reads. Unlike a naive chunk cache
1364
+ (whose misses re-read everything), a miss here
1365
+ is one pread served by the kernel page cache — amplification is a
1366
+ memcpy, and the OrderedDict LRU evicts one block, not everything.'''
1367
+ off = rec_no * self._rec_size
1368
+ bno, boff = divmod(off, self.BLOCK)
1369
+ cache = self._blk_cache
1370
+ blk = cache.get(bno)
1371
+ if blk is None:
1372
+ want = min(self.BLOCK, self._mft_size - bno * self.BLOCK)
1373
+ if want <= 0:
1374
+ return None
1375
+ if len(cache) >= 512: # 32 MiB cap
1376
+ cache.popitem(last=False)
1377
+ blk = cache[bno] = self._mft_bulk(bno * self.BLOCK, want)
1378
+ else:
1379
+ cache.move_to_end(bno)
1380
+ rec = blk[boff:boff + self._rec_size]
1381
+ return rec if len(rec) == self._rec_size else None
1382
+
1383
+ # -- raw attribute access (follows $ATTRIBUTE_LIST) --
1384
+
1385
+ def _record_attrs(self, rec_no: int) -> list:
1386
+ memo = self._attr_memo
1387
+ got = memo.get(rec_no)
1388
+ if got is None:
1389
+ if len(memo) >= 2048:
1390
+ memo.clear()
1391
+ got = memo[rec_no] = list(self._record_attrs_uncached(rec_no))
1392
+ return got
1393
+
1394
+ def _record_attrs_uncached(self, rec_no: int):
1395
+ '''Yield (frozen_record, attr_off, attr_len) for every attribute of
1396
+ this inode, base record first, then extension records in $ATTRIBUTE_
1397
+ LIST order (each extension yields its own attributes).'''
1398
+ rec = self._load_record(rec_no)
1399
+ if rec is None:
1400
+ raise NtfsError(errno.EIO, f'record {rec_no} unreadable or torn')
1401
+ frozen = bytes(rec)
1402
+ attr_list = None
1403
+ for a, t, length in _attrs(frozen):
1404
+ if t == AT_ATTRIBUTE_LIST:
1405
+ attr_list = (frozen, a, length)
1406
+ yield (frozen, a, length)
1407
+ if attr_list is None:
1408
+ return
1409
+ frozen0, a, length = attr_list
1410
+ if frozen0[a + 8] == 0: # resident list value
1411
+ v_len = struct.unpack_from('<I', frozen0, a + 16)[0]
1412
+ v_ofs = struct.unpack_from('<H', frozen0, a + 20)[0]
1413
+ listing = frozen0[a + v_ofs:a + v_ofs + v_len]
1414
+ else:
1415
+ _p, _ph, runs = _check_mapping_pairs(frozen0, a, length, self.nr_clusters())
1416
+ size = struct.unpack_from('<q', frozen0, a + 48)[0]
1417
+ listing = self._runs_read(runs, 0, size)
1418
+ pos, seen = 0, set()
1419
+ while pos + 26 <= len(listing):
1420
+ e_len = struct.unpack_from('<H', listing, pos + 4)[0]
1421
+ if e_len < 26:
1422
+ break
1423
+ mref = struct.unpack_from('<Q', listing, pos + 16)[0] & MREF_MASK
1424
+ if mref != rec_no and mref not in seen:
1425
+ seen.add(mref)
1426
+ ext = self._load_record(mref)
1427
+ if ext is not None:
1428
+ eb = bytes(ext)
1429
+ for ea, _et, el in _attrs(eb):
1430
+ yield (eb, ea, el)
1431
+ pos += e_len
1432
+
1433
+ def _stream_runs(self, rec_no: int, attr_type: int, name: bytes | None):
1434
+ '''(data_size, vcn-ordered runs) of a non-resident attribute, pieces
1435
+ merged across extension records. Raises ENOENT when absent.'''
1436
+ data_size, runs, found = None, [], False
1437
+ for frozen, a, length in self._record_attrs(rec_no):
1438
+ if struct.unpack_from('<I', frozen, a)[0] != attr_type:
1439
+ continue
1440
+ nlen = frozen[a + 9]
1441
+ nofs = struct.unpack_from('<H', frozen, a + 10)[0]
1442
+ aname = frozen[a + nofs:a + nofs + 2 * nlen] if nlen else None
1443
+ if (name or None) != (aname or None):
1444
+ continue
1445
+ found = True
1446
+ if frozen[a + 8] == 0:
1447
+ return None, None # resident — caller reads the value instead
1448
+ if struct.unpack_from('<q', frozen, a + 16)[0] == 0: # lowest_vcn
1449
+ data_size = struct.unpack_from('<q', frozen, a + 48)[0]
1450
+ _p, _ph, r = _check_mapping_pairs(frozen, a, length, self.nr_clusters())
1451
+ runs.extend(r)
1452
+ if not found:
1453
+ raise NtfsError(errno.ENOENT, f'record {rec_no}: attr {attr_type:#x} absent')
1454
+ runs.sort(key=lambda t: t[2])
1455
+ return data_size if data_size is not None else 0, runs
1456
+
1457
+ def _attr_value(self, rec_no: int, attr_type: int, name: bytes | None) -> bytes:
1458
+ for frozen, a, _length in self._record_attrs(rec_no):
1459
+ if struct.unpack_from('<I', frozen, a)[0] != attr_type:
1460
+ continue
1461
+ nlen = frozen[a + 9]
1462
+ nofs = struct.unpack_from('<H', frozen, a + 10)[0]
1463
+ aname = frozen[a + nofs:a + nofs + 2 * nlen] if nlen else None
1464
+ if (name or None) != (aname or None):
1465
+ continue
1466
+ if frozen[a + 8] == 0:
1467
+ v_len = struct.unpack_from('<I', frozen, a + 16)[0]
1468
+ v_ofs = struct.unpack_from('<H', frozen, a + 20)[0]
1469
+ return frozen[a + v_ofs:a + v_ofs + v_len]
1470
+ size, runs = self._stream_runs(rec_no, attr_type, name)
1471
+ return self._runs_read(runs, 0, size)
1472
+ raise NtfsError(errno.ENOENT, f'record {rec_no}: attr {attr_type:#x} absent')
1473
+
1474
+ def _read_whole_attr(self, mft_no: int, attr_type: int,
1475
+ name=None, name_len: int = 0) -> bytes:
1476
+ return self._attr_value(mft_no, attr_type, name)
1477
+
1478
+ # -- directories: raw $I30 walk + path resolution --
1479
+
1480
+ I30 = '$I30'.encode('utf-16-le')
1481
+
1482
+ def _dir_entries(self, mft_no: int) -> list[dict]:
1483
+ root = self._attr_value(mft_no, AT_INDEX_ROOT, self.I30)
1484
+ alloc = bmp = None
1485
+ try:
1486
+ alloc = self._attr_value(mft_no, AT_INDEX_ALLOCATION, self.I30)
1487
+ bmp = self._attr_value(mft_no, AT_BITMAP, self.I30)
1488
+ except NtfsError:
1489
+ pass
1490
+ pairs, problems = _view_index_entries(root, alloc, bmp, f'mft#{mft_no}',
1491
+ dir_index=True)
1492
+ if problems:
1493
+ raise NtfsError(errno.EIO, f'$I30 of record {mft_no}: {problems[0]}')
1494
+ entries = []
1495
+ for key, mref in pairs:
1496
+ if len(key) < 66:
1497
+ continue
1498
+ n_len, n_type = key[64], key[65]
1499
+ attrs = struct.unpack_from('<I', key, 56)[0]
1500
+ entries.append({'name': key[66:66 + 2 * n_len].decode('utf-16-le', 'replace'),
1501
+ 'name_type': n_type, 'mref': mref,
1502
+ 'dt_type': NTFS_DT_DIR if attrs & 0x10000000 else 8})
1503
+ return entries
1504
+
1505
+ def _resolve(self, path: str) -> int:
1506
+ cur = FILE_ROOT
1507
+ for comp in path.strip('/').split('/'):
1508
+ if not comp:
1509
+ continue
1510
+ for e in self._dir_entries(cur):
1511
+ if self.names_equal(e['name'], comp):
1512
+ cur = e['mref'] & MREF_MASK
1513
+ break
1514
+ else:
1515
+ raise NtfsError(errno.ENOENT, f'{comp!r} not found resolving {path!r}')
1516
+ return cur
1517
+
1518
+ def scan_dir_no(self, mft_no: int) -> list[dict]:
1519
+ entries = self._dir_entries(mft_no)
1520
+ for entry in entries:
1521
+ entry['record'] = entry['mref'] & MREF_MASK
1522
+ entry['seq'] = entry['mref'] >> 48
1523
+ entry['status'] = self._entry_status(mft_no, entry)
1524
+ return entries
1525
+
1526
+ def scan_dir(self, path: str) -> list[dict]:
1527
+ return self.scan_dir_no(self._resolve(path))
1528
+
1529
+ def resolve(self, path: str) -> int:
1530
+ return self._resolve(path)
1531
+
1532
+ def record_info(self, record_no: int) -> dict:
1533
+ try:
1534
+ attrs = list(self._record_attrs(record_no & MREF_MASK))
1535
+ except NtfsError as exc:
1536
+ return {'open': False, 'errno': exc.errno or errno.EIO, 'error': str(exc)}
1537
+ frozen = attrs[0][0] if attrs else self.read_record(record_no & MREF_MASK)
1538
+ info = {'open': True,
1539
+ 'seq': struct.unpack_from('<H', frozen, 16)[0],
1540
+ 'link_count': struct.unpack_from('<H', frozen, 18)[0],
1541
+ 'in_use': bool(struct.unpack_from('<H', frozen, 22)[0] & 1),
1542
+ 'is_dir': bool(struct.unpack_from('<H', frozen, 22)[0] & 2),
1543
+ 'names': []}
1544
+ for rec, a, _length in attrs:
1545
+ if struct.unpack_from('<I', rec, a)[0] != AT_FILE_NAME or rec[a + 8]:
1546
+ continue
1547
+ v_len = struct.unpack_from('<I', rec, a + 16)[0]
1548
+ v_ofs = struct.unpack_from('<H', rec, a + 20)[0]
1549
+ fn = rec[a + v_ofs:a + v_ofs + v_len]
1550
+ if len(fn) < 66:
1551
+ continue
1552
+ parent = struct.unpack_from('<Q', fn, 0)[0]
1553
+ info['names'].append({
1554
+ 'name': fn[66:66 + 2 * fn[64]].decode('utf-16-le', 'replace'),
1555
+ 'type': fn[65], 'parent_record': parent & MREF_MASK,
1556
+ 'parent_seq': parent >> 48})
1557
+ return info
1558
+
1559
+ def classify_dirent(self, path: str, name: str) -> dict:
1560
+ dir_no = self._resolve(path)
1561
+ for e in self._dir_entries(dir_no):
1562
+ if self.names_equal(e['name'], name):
1563
+ found = {'name': name, 'record': e['mref'] & MREF_MASK,
1564
+ 'seq': e['mref'] >> 48}
1565
+ return self._classify_found(dir_no, name, found)
1566
+ raise NtfsError(errno.ENOENT, f'index entry {name!r} not found in {path!r}')
1567
+
1568
+ def remove_dirent(self, path: str, name: str, really: bool) -> dict:
1569
+ if really:
1570
+ raise NtfsError(0, 'RawVolume is read-only — use RawVolumeRW')
1571
+ dir_no = self._resolve(path)
1572
+ for e in self._dir_entries(dir_no):
1573
+ if self.names_equal(e['name'], name):
1574
+ return {'name': name, 'record': e['mref'] & MREF_MASK,
1575
+ 'seq': e['mref'] >> 48}
1576
+ raise NtfsError(errno.ENOENT, f'index entry {name!r} not found in {path!r}')
1577
+
1578
+ def usn_check(self, path: str = '/$Extend/$UsnJrnl') -> dict:
1579
+ info = {'present': False, 'problems': [], 'notes': [], 'records': 0,
1580
+ 'journal_id': None, 'lowest': None, 'next_usn': None}
1581
+ try:
1582
+ jrnl = self._resolve(path)
1583
+ except NtfsError:
1584
+ return info
1585
+ info['present'] = True
1586
+ problems = info['problems']
1587
+ lowest = 0
1588
+ try:
1589
+ blob = self._attr_value(jrnl, AT_DATA, '$Max'.encode('utf-16-le'))
1590
+ except NtfsError:
1591
+ problems.append('$Max stream missing')
1592
+ blob = None
1593
+ if blob is not None:
1594
+ if len(blob) != 32:
1595
+ problems.append(f'$Max is {len(blob)} bytes, expected 32')
1596
+ else:
1597
+ max_size, _d, jid, lowest = struct.unpack('<QQQQ', blob)
1598
+ info.update(journal_id=jid, lowest=lowest)
1599
+ if not 0 < max_size < 1 << 40:
1600
+ problems.append(f'implausible journal MaximumSize {max_size}')
1601
+ jn = '$J'.encode('utf-16-le')
1602
+ try:
1603
+ data_size, runs = self._stream_runs(jrnl, AT_DATA, jn)
1604
+ if data_size is None: # resident $J
1605
+ value = self._attr_value(jrnl, AT_DATA, jn)
1606
+ data_size, runs = len(value), None
1607
+ except NtfsError:
1608
+ problems.append('$J stream missing')
1609
+ return info
1610
+ info['next_usn'] = data_size
1611
+ if lowest > data_size:
1612
+ problems.append(f'LowestValidUsn {lowest} beyond journal end {data_size}')
1613
+ return info
1614
+ if lowest & 7:
1615
+ problems.append(f'LowestValidUsn {lowest} not 8-byte aligned')
1616
+ return info
1617
+ start = lowest
1618
+ if runs:
1619
+ first_alloc = runs[0][2] * self.cluster_size()
1620
+ if first_alloc > start:
1621
+ info['notes'].append(
1622
+ f'LowestValidUsn {start} lags the first allocated byte '
1623
+ f'{first_alloc} (purged region)')
1624
+ start = first_alloc
1625
+ read = lambda pos, count: self._runs_read(runs, pos, count)
1626
+ else:
1627
+ read = lambda pos, count: value[pos:pos + count]
1628
+ count, walk_problems = _walk_usn_records(read, start, data_size)
1629
+ info['records'] = count
1630
+ problems.extend(walk_problems)
1631
+ return info
1632
+
1633
+
1634
+ class _IndexSchema:
1635
+ '''How one NTFS index type names itself, orders its keys, and frames its
1636
+ entries — the B+ write engine is otherwise generic over it. $I30
1637
+ (directory indexes) is the reference schema; the $Secure view indexes
1638
+ $SII/$SDH will plug in the same way.
1639
+
1640
+ name UTF-16LE attribute name ($I30, $SII, $SDH)
1641
+ collation the COLLATION_* rule id (goes in the $INDEX_ROOT header)
1642
+ sort_key stored-key bytes -> a Python-comparable ordering key
1643
+ build_leaf (value, key_bytes) -> a leaf INDEX_ENTRY
1644
+
1645
+ An INDEX_ENTRY stores its key as `key_length` bytes at offset 16; the
1646
+ collation key is derived from those bytes by `sort_key` (for $I30 the
1647
+ stored key is the whole $FILE_NAME and the collation key is its upcased
1648
+ name; for a view index the stored key is the collation key directly).'''
1649
+
1650
+ __slots__ = ('name', 'collation', 'sort_key', 'build_leaf')
1651
+
1652
+ def __init__(self, name, collation, sort_key, build_leaf):
1653
+ self.name = name
1654
+ self.collation = collation
1655
+ self.sort_key = sort_key
1656
+ self.build_leaf = build_leaf
1657
+
1658
+ @staticmethod
1659
+ def stored_key(entry) -> bytes:
1660
+ return bytes(entry[16:16 + struct.unpack_from('<H', entry, 10)[0]])
1661
+
1662
+ def entry_sort_key(self, entry):
1663
+ return self.sort_key(self.stored_key(entry))
1664
+
1665
+
1666
+ class RawVolumeRW(RawVolume):
1667
+ ''' The native write engine: sealed record writes, $MFTMirr sync, dirty-flag
1668
+ lifecycle, allocators, B+ tree index edits (remove / insert with splits /
1669
+ bulk-load rebuild), $Secure and USN repairs '''
1670
+
1671
+ def __init__(self, device: str, base: int = 0):
1672
+ super().__init__(device, readonly=True, base=base)
1673
+ os.close(self._fd)
1674
+ self._fd = os.open(device, os.O_RDWR | _O_BINARY)
1675
+ self._mirror = None
1676
+ self._was_dirty = self._dirty_flag()
1677
+ if not self._was_dirty:
1678
+ self._set_dirty(True) # crash mid-op leaves the volume flagged
1679
+
1680
+ def close(self) -> None:
1681
+ if getattr(self, '_fd', None) is not None and not self._was_dirty:
1682
+ self._set_dirty(False) # clean close: clear only what we set
1683
+ super().close()
1684
+
1685
+ # -- dirty flag write side ($VOLUME_INFORMATION probe lives in RawVolume) --
1686
+
1687
+ def _set_dirty(self, on: bool) -> None:
1688
+ rec, off = self._volume_info()
1689
+ flags = struct.unpack_from('<H', rec, off)[0]
1690
+ struct.pack_into('<H', rec, off, flags | 1 if on else flags & ~1)
1691
+ self.write_record(3, rec)
1692
+
1693
+ # -- write plumbing --
1694
+
1695
+ def _runs_write(self, runs, offset: int, data: bytes) -> None:
1696
+ csz = self._cluster_size
1697
+ end, written = offset + len(data), 0
1698
+ for lcn, run_len, vcn in runs:
1699
+ r_start = vcn * csz
1700
+ if r_start >= end:
1701
+ break
1702
+ r_end = r_start + run_len * csz
1703
+ lo, hi = max(offset, r_start), min(end, r_end)
1704
+ if lo >= hi:
1705
+ continue
1706
+ chunk = data[lo - offset:hi - offset]
1707
+ if _pwrite(self._fd, chunk, self._base + lcn * csz + (lo - r_start)) != len(chunk):
1708
+ raise NtfsError(errno.EIO, f'short write at stream offset {lo}')
1709
+ written += len(chunk)
1710
+ if written != len(data):
1711
+ raise NtfsError(errno.EIO, f'write spans a hole ({written}/{len(data)})')
1712
+
1713
+ def _mirror_runs(self):
1714
+ if self._mirror is None:
1715
+ self._mirror = self._stream_runs(1, AT_DATA, None)[1] # $MFTMirr
1716
+ return self._mirror
1717
+
1718
+ def write_record(self, rec_no: int, logical) -> None:
1719
+ '''Seal fixups over LOGICAL (fixed-up) record content and write it.
1720
+ Records 0-3 are mirrored into $MFTMirr, as the driver verifies.'''
1721
+ rec = bytearray(logical)
1722
+ if rec[:4] != b'FILE' or len(rec) != self._rec_size:
1723
+ raise NtfsError(errno.EINVAL, 'refusing to write a non-FILE record')
1724
+ _seal_fixups(rec)
1725
+ blob = bytes(rec)
1726
+ self._runs_write(self._mft_runs, rec_no * self._rec_size, blob)
1727
+ if rec_no < 4:
1728
+ self._runs_write(self._mirror_runs(), rec_no * self._rec_size, blob)
1729
+ self._blk_cache.clear()
1730
+ self._attr_memo.clear()
1731
+
1732
+ # -- phase 2: allocators ($Bitmap clusters + $MFT record slots) --
1733
+
1734
+ def _bitmap_state(self):
1735
+ if getattr(self, '_bm', None) is None:
1736
+ size, runs = self._stream_runs(FILE_BITMAP, AT_DATA, None)
1737
+ self._bm = bytearray(self._runs_read(runs, 0, size))
1738
+ self._bm_runs = runs
1739
+ return self._bm
1740
+
1741
+ def _bitmap_flush(self, bits) -> None:
1742
+ lo, hi = min(bits) >> 3, (max(bits) >> 3) + 1
1743
+ self._runs_write(self._bm_runs, lo, bytes(self._bm[lo:hi]))
1744
+
1745
+ def apply_bitmap(self, audit: dict) -> None:
1746
+ '''Rewrite $Bitmap with the computed truth (the chkdsk stage-5 fix).'''
1747
+ if audit['failures']:
1748
+ raise NtfsError(0, 'refusing to write $Bitmap: usage map is incomplete')
1749
+ nc = audit['nr_clusters']
1750
+ data = bytearray(audit['used_bytes'])
1751
+ if nc & 7: # preserve the on-disk padding-bit convention in the tail
1752
+ data[-1] |= audit['ondisk_tail'] & (0xFF ^ ((1 << (nc & 7)) - 1))
1753
+ self._bitmap_state() # populate self._bm_runs
1754
+ self._runs_write(self._bm_runs, 0, bytes(data))
1755
+ self._bm = None # drop the cached map; reload lazily
1756
+
1757
+ def secure_fix_mirrors(self, fixes: list) -> int:
1758
+ '''Copy the hash-verified side of each mismatched $SDS entry pair over
1759
+ the corrupt side (both copies end up canonical: offset = primary).'''
1760
+ name = '$SDS'.encode('utf-16-le')
1761
+ size, runs = self._stream_runs(FILE_SECURE, AT_DATA, name)
1762
+ done = 0
1763
+ for pos, length, source in fixes:
1764
+ src = pos if source == 'primary' else pos + SDS_BLOCK
1765
+ blob = bytearray(self._runs_read(runs, src, length))
1766
+ struct.pack_into('<Q', blob, 8, pos) # canonical offset field
1767
+ for dst in (pos, pos + SDS_BLOCK):
1768
+ self._runs_write(runs, dst, bytes(blob))
1769
+ done += 1
1770
+ return done
1771
+
1772
+ def usn_reset(self, path: str = '/$Extend/$UsnJrnl') -> int:
1773
+ '''The chkdsk/fsutil-style journal reset: truncate $J to zero (freeing
1774
+ its clusters) and stamp $Max with a fresh journal id and
1775
+ LowestValidUsn 0. Consumers detect the id change and rescan.
1776
+ Returns the new journal id.'''
1777
+ jrnl = self._resolve(path)
1778
+ jn = '$J'.encode('utf-16-le')
1779
+ found = self._find_attr(jrnl, AT_DATA, jn)
1780
+ if found is None:
1781
+ raise NtfsError(errno.ENOENT, 'open $J')
1782
+ rno, rec, a = found
1783
+ rec = bytearray(rec)
1784
+ if rec[a + 8] == 1: # non-resident: free + zero
1785
+ length = struct.unpack_from('<I', rec, a + 4)[0]
1786
+ _p, _ph, r = _check_mapping_pairs(rec, a, length, self.nr_clusters())
1787
+ runs = [(lcn, rlen) for lcn, rlen, _v in r]
1788
+ struct.pack_into('<q', rec, a + 0x10, 0) # lowest_vcn
1789
+ struct.pack_into('<q', rec, a + 0x18, -1) # highest_vcn (0 clusters)
1790
+ struct.pack_into('<Q', rec, a + 0x28, 0) # allocated_size
1791
+ struct.pack_into('<Q', rec, a + 0x30, 0) # data_size
1792
+ struct.pack_into('<Q', rec, a + 0x38, 0) # initialized_size
1793
+ mp_off = struct.unpack_from('<H', rec, a + 0x20)[0]
1794
+ rec[a + mp_off] = 0 # runlist terminator
1795
+ self.write_record(rno, rec) # clear ref before freeing
1796
+ if runs:
1797
+ self.free_clusters(runs)
1798
+ else: # resident: empty the value
1799
+ self._replace_resident_value(rec, a, b'')
1800
+ self.write_record(rno, rec)
1801
+
1802
+ mn = '$Max'.encode('utf-16-le')
1803
+ try:
1804
+ old = self._attr_value(jrnl, AT_DATA, mn)
1805
+ except NtfsError:
1806
+ old = b''
1807
+ max_size, delta = struct.unpack_from('<QQ', old) if len(old) >= 16 else (0, 0)
1808
+ if not 0 < max_size < 1 << 40 or not 0 < delta < 1 << 40:
1809
+ max_size, delta = 32 * 1024 * 1024, 8 * 1024 * 1024
1810
+ new_id = int((time.time() + 11644473600) * 10 ** 7) # NTFS FILETIME
1811
+ blob = struct.pack('<QQQQ', max_size, delta, new_id, 0)
1812
+ mfound = self._find_attr(jrnl, AT_DATA, mn)
1813
+ if mfound is None:
1814
+ raise NtfsError(errno.ENOENT, 'open $Max')
1815
+ mrno, mrec, ma = mfound
1816
+ mrec = bytearray(mrec)
1817
+ if mrec[ma + 8] == 0: # resident (typical)
1818
+ self._replace_resident_value(mrec, ma, blob)
1819
+ self.write_record(mrno, mrec)
1820
+ else: # non-resident: overwrite
1821
+ _size, mruns = self._stream_runs(jrnl, AT_DATA, mn)
1822
+ self._runs_write(mruns, 0, blob)
1823
+ return new_id
1824
+
1825
+ def rebuild_secure_indexes(self, sds_entries: dict) -> dict:
1826
+ '''Native $SII/$SDH rebuild: wipe each index and re-add one entry per
1827
+ $SDS descriptor. `sds_entries` maps security_id -> (hash, sds_offset,
1828
+ length). Bulk-loads a resident root, a single INDX block, or a
1829
+ multi-level tree as needed (via the shared _bulk_write_index engine),
1830
+ with the two view collations — $SII by security-id (ULONG), $SDH by
1831
+ (hash, id) (SECURITY_HASH).'''
1832
+ orders = (
1833
+ (self._sii, sorted(sds_entries),
1834
+ lambda sid: struct.pack('<I', sid)),
1835
+ (self._sdh, sorted(sds_entries, key=lambda i: (sds_entries[i][0], i)),
1836
+ lambda sid: struct.pack('<II', sds_entries[sid][0], sid)))
1837
+ built = 0
1838
+ for schema, order, key_of in orders:
1839
+ entries = []
1840
+ for sid in order:
1841
+ h, off, length = sds_entries[sid]
1842
+ data = struct.pack('<IIQI', h, sid, off, length)
1843
+ entries.append(schema.build_leaf(data, key_of(sid)))
1844
+ self._write_view_index(FILE_SECURE, schema, entries)
1845
+ built += len(entries)
1846
+ return {'added': built}
1847
+
1848
+ def _write_view_index(self, owner: int, schema, entries: list) -> None:
1849
+ '''Replace `schema`'s index on `owner` with `entries` (already in
1850
+ collation order) — the $SII/$SDH ($Secure) analogue of the $I30
1851
+ rebuild: resident root, single INDX block, or a bulk-loaded
1852
+ multi-level tree, all through the shared _bulk_write_index engine.'''
1853
+ self._ix_override = schema
1854
+ try:
1855
+ self._bulk_write_index(owner, entries)
1856
+ finally:
1857
+ self._ix_override = None
1858
+
1859
+ def alloc_clusters(self, count: int, near_lcn: int = 0) -> list[tuple[int, int]]:
1860
+ '''First-fit from the hint (byte-skipping full bytes), wrapping once.
1861
+ Sets the bits, flushes $Bitmap, returns [(lcn, run_len)] runs. The
1862
+ stage-5 audit is the oracle: freshly allocated, unreferenced
1863
+ clusters must show up as exactly `count` extra bits.'''
1864
+ bm = self._bitmap_state()
1865
+ nc = self.nr_clusters()
1866
+ picked: list[int] = []
1867
+ c = min(max(near_lcn, 0), nc - 1)
1868
+ scanned = 0
1869
+ while len(picked) < count and scanned <= nc:
1870
+ if c >= nc:
1871
+ c = 0
1872
+ if not c & 7 and bm[c >> 3] == 0xFF and c + 8 <= nc:
1873
+ c += 8
1874
+ scanned += 8
1875
+ continue
1876
+ if not bm[c >> 3] & (1 << (c & 7)):
1877
+ picked.append(c)
1878
+ c += 1
1879
+ scanned += 1
1880
+ if len(picked) < count:
1881
+ raise NtfsError(errno.ENOSPC, 'volume full')
1882
+ for c in picked:
1883
+ bm[c >> 3] |= 1 << (c & 7)
1884
+ self._bitmap_flush(picked)
1885
+ picked.sort()
1886
+ runs, start, prev = [], picked[0], picked[0]
1887
+ for c in picked[1:]:
1888
+ if c != prev + 1:
1889
+ runs.append((start, prev - start + 1))
1890
+ start = c
1891
+ prev = c
1892
+ runs.append((start, prev - start + 1))
1893
+ return runs
1894
+
1895
+ def free_clusters(self, runs) -> None:
1896
+ bm = self._bitmap_state()
1897
+ bits, seen = [], set()
1898
+ for lcn, run_len in runs:
1899
+ for c in range(lcn, lcn + run_len):
1900
+ if c in seen or not bm[c >> 3] & (1 << (c & 7)):
1901
+ raise NtfsError(errno.EIO, f'double free of cluster {c}')
1902
+ seen.add(c)
1903
+ bits.append(c)
1904
+ for c in bits:
1905
+ bm[c >> 3] &= ~(1 << (c & 7))
1906
+ self._bitmap_flush(bits)
1907
+
1908
+ def _mft_bitmap(self):
1909
+ '''(bitmap_bytes, writer) for $MFT's own $BITMAP — resident on small
1910
+ volumes (rewrite record 0, mirrored), non-resident on real ones.'''
1911
+ for frozen, a, _l in self._record_attrs(0):
1912
+ if (struct.unpack_from('<I', frozen, a)[0] != AT_BITMAP
1913
+ or frozen[a + 9] != 0):
1914
+ continue # unnamed only
1915
+ if frozen[a + 8] == 0:
1916
+ v_len = struct.unpack_from('<I', frozen, a + 16)[0]
1917
+ v_ofs = struct.unpack_from('<H', frozen, a + 20)[0]
1918
+ blob = frozen[a + v_ofs:a + v_ofs + v_len]
1919
+
1920
+ def write_res(data: bytes, a=a, v_ofs=v_ofs, v_len=v_len):
1921
+ rec = self._load_record(0)
1922
+ assert rec is not None
1923
+ rec[a + v_ofs:a + v_ofs + v_len] = data
1924
+ self.write_record(0, rec)
1925
+ return blob, write_res
1926
+ _p, _ph, runs = _check_mapping_pairs(frozen, a,
1927
+ struct.unpack_from('<I', frozen,
1928
+ a + 4)[0],
1929
+ self.nr_clusters())
1930
+ size = struct.unpack_from('<q', frozen, a + 48)[0]
1931
+ blob = self._runs_read(runs, 0, size)
1932
+
1933
+ def write_nonres(data: bytes, runs=runs):
1934
+ self._runs_write(runs, 0, data)
1935
+ return blob, write_nonres
1936
+ raise NtfsError(errno.ENOENT, "$MFT's $BITMAP not found")
1937
+
1938
+ def alloc_record(self) -> int:
1939
+ '''Reuse a free MFT record slot (bit clear AND record not in use);
1940
+ growing the MFT is deliberately unimplemented — ENOSPC instead.'''
1941
+ bm, write = self._mft_bitmap()
1942
+ new = bytearray(bm)
1943
+ for rec_no in range(24, self._mft_size // self._rec_size):
1944
+ if new[rec_no >> 3] & (1 << (rec_no & 7)):
1945
+ continue
1946
+ rec = self._load_record(rec_no)
1947
+ if rec is not None and struct.unpack_from('<H', rec, 22)[0] & 1:
1948
+ continue # bitmap stale: record actually live — never take it
1949
+ new[rec_no >> 3] |= 1 << (rec_no & 7)
1950
+ write(bytes(new))
1951
+ return rec_no
1952
+ raise NtfsError(errno.ENOSPC, 'no free MFT records (growth unimplemented)')
1953
+
1954
+ def apply_mft_bitmap(self, used: bytes, nbits: int) -> int:
1955
+ '''Rewrite $MFT's $BITMAP so bit i matches record i's surveyed in-use
1956
+ flag, for records [0, nbits); bits past nbits keep their on-disk
1957
+ value. The crash shape this repairs: alloc_record's bitmap write
1958
+ reached the disk but the record write never did — a leaked set bit
1959
+ that chkdsk reports as "the MFT BITMAP attribute is incorrect".'''
1960
+ bm, write = self._mft_bitmap()
1961
+ new = bytearray(bm)
1962
+ changed = 0
1963
+ for i in range(min(nbits, len(new) * 8)):
1964
+ want = bool(used[i >> 3] & (1 << (i & 7)))
1965
+ if bool(new[i >> 3] & (1 << (i & 7))) != want:
1966
+ changed += 1
1967
+ if want:
1968
+ new[i >> 3] |= 1 << (i & 7)
1969
+ else:
1970
+ new[i >> 3] &= ~(1 << (i & 7)) & 0xFF
1971
+ if changed:
1972
+ write(bytes(new))
1973
+ return changed
1974
+
1975
+ def free_record(self, rec_no: int) -> None:
1976
+ bm, write = self._mft_bitmap()
1977
+ if not bm[rec_no >> 3] & (1 << (rec_no & 7)):
1978
+ raise NtfsError(errno.EIO, f'record {rec_no} already free in bitmap')
1979
+ new = bytearray(bm)
1980
+ new[rec_no >> 3] &= ~(1 << (rec_no & 7))
1981
+ write(bytes(new))
1982
+
1983
+ # -- phase 3/3b: index-entry removal (leaf splice + internal promotion) --
1984
+
1985
+ def _write_index_block(self, ia_runs, block_index: int, block_size: int,
1986
+ blk: bytearray) -> None:
1987
+ _seal_fixups(blk)
1988
+ self._runs_write(ia_runs, block_index * block_size, bytes(blk))
1989
+ self._blk_cache.clear()
1990
+ self._attr_memo.clear()
1991
+
1992
+ @staticmethod
1993
+ def _find_entry(buf, hdr_off: int, matches):
1994
+ '''(pos, length, flags) of the first non-END entry for which
1995
+ matches(name) is true, else None. Does not mutate.'''
1996
+ entries_ofs, index_len = struct.unpack_from('<II', buf, hdr_off)
1997
+ pos, limit = hdr_off + entries_ofs, hdr_off + index_len
1998
+ while pos + 16 <= limit:
1999
+ length = struct.unpack_from('<H', buf, pos + 8)[0]
2000
+ flags = struct.unpack_from('<H', buf, pos + 12)[0]
2001
+ if length < 16 or pos + length > limit:
2002
+ return None
2003
+ if not flags & 2:
2004
+ n_len = buf[pos + 16 + 64]
2005
+ nm = bytes(buf[pos + 16 + 66:pos + 16 + 66 + 2 * n_len]).decode(
2006
+ 'utf-16-le', 'replace')
2007
+ if matches(nm):
2008
+ return pos, length, flags
2009
+ if flags & 2:
2010
+ break
2011
+ pos += length
2012
+ return None
2013
+
2014
+
2015
+ @staticmethod
2016
+ def _splice_at(buf, hdr_off: int, pos: int, length: int) -> None:
2017
+ ''' Remove the entry at pos '''
2018
+
2019
+ _eo, index_len = struct.unpack_from('<II', buf, hdr_off)
2020
+ limit = hdr_off + index_len
2021
+ tail = bytes(buf[pos + length:limit])
2022
+ buf[pos:pos + len(tail)] = tail
2023
+ new_len = index_len - length
2024
+ for i in range(hdr_off + new_len, limit):
2025
+ buf[i] = 0
2026
+ struct.pack_into('<I', buf, hdr_off + 4, new_len)
2027
+
2028
+
2029
+ def _read_indx_by_vcn(self, alloc, block_size, vcn):
2030
+ '''(block_index, fixed-up block) whose header VCN == vcn, matched by the
2031
+ block's own index_block_vcn field (VCN-unit-agnostic) '''
2032
+
2033
+ for i in range(len(alloc) // block_size):
2034
+ if alloc[i * block_size:i * block_size + 4] != b'INDX':
2035
+ continue
2036
+ if struct.unpack_from('<q', alloc, i * block_size + 16)[0] != vcn:
2037
+ continue
2038
+ blk = bytearray(alloc[i * block_size:(i + 1) * block_size])
2039
+ if _apply_fixups(blk):
2040
+ return i, blk
2041
+ return None, None
2042
+
2043
+
2044
+ def _rightmost_leaf(self, alloc, block_size, start_vcn):
2045
+ '''Descend the rightmost (END-entry) subnode pointers from start_vcn to
2046
+ a leaf block — the subtree's maximum lives there.'''
2047
+ vcn, seen = start_vcn, set()
2048
+ while True:
2049
+ if vcn in seen:
2050
+ raise NtfsError(errno.EIO, 'cyclic index subnode chain')
2051
+ seen.add(vcn)
2052
+ i, blk = self._read_indx_by_vcn(alloc, block_size, vcn)
2053
+ if blk is None:
2054
+ raise NtfsError(errno.EIO, f'index block VCN {vcn} unreadable')
2055
+ eo, il = struct.unpack_from('<II', blk, 24)
2056
+ pos, end = 24 + eo, 24 + il
2057
+ while pos + 16 <= end:
2058
+ length = struct.unpack_from('<H', blk, pos + 8)[0]
2059
+ flags = struct.unpack_from('<H', blk, pos + 12)[0]
2060
+ if length < 16:
2061
+ raise NtfsError(errno.EIO, 'corrupt index node')
2062
+ if flags & 2: # END entry
2063
+ if flags & 1: # has a subnode — descend rightmost
2064
+ vcn = struct.unpack_from('<q', blk, pos + length - 8)[0]
2065
+ break
2066
+ return i, blk # leaf
2067
+ pos += length
2068
+ else:
2069
+ raise NtfsError(errno.EIO, 'index node without END entry')
2070
+
2071
+
2072
+ @staticmethod
2073
+ def _last_real_entry(buf, hdr_off):
2074
+ '''(pos, length) of the last non-END entry — the node's maximum key.'''
2075
+ eo, il = struct.unpack_from('<II', buf, hdr_off)
2076
+ pos, limit, last = hdr_off + eo, hdr_off + il, None
2077
+ while pos + 16 <= limit:
2078
+ length = struct.unpack_from('<H', buf, pos + 8)[0]
2079
+ flags = struct.unpack_from('<H', buf, pos + 12)[0]
2080
+ if length < 16:
2081
+ break
2082
+ if flags & 2:
2083
+ break
2084
+ last = (pos, length)
2085
+ pos += length
2086
+ return last
2087
+
2088
+
2089
+ def _remove_internal(self, node, hdr_off, ie_pos, ie_len, write_node,
2090
+ ia_runs, block_size, alloc) -> None:
2091
+ '''Delete an internal (subnode-bearing) entry by promoting its in-order
2092
+ predecessor — the maximum key of its left subtree — into its slot,
2093
+ then removing that predecessor from its leaf. Faithful to a B-tree
2094
+ internal delete; no rebalancing (underfull nodes stay valid).
2095
+
2096
+ Order matters for crash safety: remove the predecessor from the leaf
2097
+ FIRST, then promote. A crash between the two leaves the target entry
2098
+ in place (repair simply re-runs) and turns the predecessor into a
2099
+ reconnectable lost file — never a duplicate key.'''
2100
+ ie_vcn = struct.unpack_from('<q', node, ie_pos + ie_len - 8)[0]
2101
+ leaf_i, leaf = self._rightmost_leaf(alloc, block_size, ie_vcn)
2102
+ pred = self._last_real_entry(leaf, 24)
2103
+ if pred is None:
2104
+ # legal shape (removals emptied the subtree) that predecessor
2105
+ # promotion cannot cross — the caller rebuilds instead
2106
+ raise NtfsError(errno.ENOTSUP, 'empty subtree under an internal entry')
2107
+ p_pos, p_len = pred
2108
+ p_bytes = bytes(leaf[p_pos:p_pos + p_len])
2109
+ p_flags = struct.unpack_from('<H', p_bytes, 12)[0]
2110
+ if p_flags & 1:
2111
+ raise NtfsError(errno.ENOTSUP, 'predecessor unexpectedly has a subnode')
2112
+
2113
+ # build the promoted separator: predecessor key/data + the deleted
2114
+ # entry's own subnode VCN, marked as a NODE entry
2115
+ key_len = struct.unpack_from('<H', p_bytes, 10)[0]
2116
+ content = p_bytes[:16 + key_len]
2117
+ content += b'\x00' * (-len(content) % 8) # 8-byte align
2118
+ repl = bytearray(content + struct.pack('<q', ie_vcn))
2119
+ struct.pack_into('<H', repl, 8, len(repl))
2120
+ struct.pack_into('<H', repl, 12, (p_flags | 1) & ~2) # NODE, not END
2121
+
2122
+ eo, index_len = struct.unpack_from('<II', node, hdr_off)
2123
+ alloc_size = struct.unpack_from('<I', node, hdr_off + 8)[0]
2124
+ new_index_len = index_len - ie_len + len(repl)
2125
+ if new_index_len > alloc_size:
2126
+ raise NtfsError(errno.ENOTSUP,
2127
+ 'promoted separator does not fit — node split needed')
2128
+
2129
+ # 1) drop the predecessor from its leaf (crash-safe half)
2130
+ self._splice_at(leaf, 24, p_pos, p_len)
2131
+ self._write_index_block(ia_runs, leaf_i, block_size, leaf)
2132
+ # 2) replace the target entry with the promoted separator
2133
+ tail = bytes(node[ie_pos + ie_len:hdr_off + index_len])
2134
+ node[ie_pos:ie_pos + len(repl)] = repl
2135
+ node[ie_pos + len(repl):ie_pos + len(repl) + len(tail)] = tail
2136
+ for i in range(hdr_off + new_index_len, hdr_off + index_len):
2137
+ node[i] = 0
2138
+ struct.pack_into('<I', node, hdr_off + 4, new_index_len)
2139
+ write_node()
2140
+
2141
+ def remove_index_entry(self, dir_no: int, name: str) -> bool:
2142
+ '''Remove one $I30 entry by name — leaf via splice, internal via
2143
+ predecessor promotion; shapes the promotion cannot cross (emptied
2144
+ subtree, overfull node) fall back to a bulk rebuild of the index
2145
+ from its own surviving entries. Raises ENOENT when absent.'''
2146
+ try:
2147
+ return self._remove_index_entry_structural(dir_no, name)
2148
+ except NtfsError as exc:
2149
+ if exc.errno != errno.ENOTSUP:
2150
+ raise
2151
+ self._rebuild_without(dir_no, name)
2152
+ return True
2153
+
2154
+ def _rebuild_without(self, dir_no: int, name: str) -> None:
2155
+ '''Bulk-rebuild the index from its own current entries minus `name` —
2156
+ always yields a canonical tree, reclaiming emptied blocks.'''
2157
+ root = self._attr_value(dir_no, AT_INDEX_ROOT, self._ix.name)
2158
+ try:
2159
+ alloc = self._attr_value(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
2160
+ except NtfsError:
2161
+ alloc = b''
2162
+ block_size = struct.unpack_from('<I', root, 8)[0]
2163
+ nodes = [(bytes(root), 16)]
2164
+ for i in range(len(alloc) // block_size):
2165
+ blk = bytearray(alloc[i * block_size:(i + 1) * block_size])
2166
+ if blk[:4] == b'INDX' and _apply_fixups(blk):
2167
+ nodes.append((bytes(blk), 24))
2168
+ items = []
2169
+ for buf, hdr in nodes:
2170
+ reals, _end = self._decode_node(buf, hdr)
2171
+ for e in reals:
2172
+ klen = struct.unpack_from('<H', e, 10)[0]
2173
+ fn = e[16:16 + klen]
2174
+ nm = self._fn_key_name(fn).decode('utf-16-le', 'replace')
2175
+ if self.names_equal(nm, name):
2176
+ continue
2177
+ mref = struct.unpack_from('<Q', e, 0)[0]
2178
+ leaf = self._ix.build_leaf(mref, fn)
2179
+ items.append((self._ix.entry_sort_key(leaf), leaf))
2180
+ items.sort(key=lambda t: t[0])
2181
+ self._bulk_write_index(dir_no, [e for _, e in items])
2182
+
2183
+ def _remove_index_entry_structural(self, dir_no: int, name: str) -> bool:
2184
+ rec = self._load_record(dir_no)
2185
+ if rec is None:
2186
+ raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
2187
+ loc = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
2188
+ if loc is None:
2189
+ raise NtfsError(errno.ENOENT, f'record {dir_no} has no $I30 index root')
2190
+ a, _ln = loc
2191
+ vofs = struct.unpack_from('<H', rec, a + 20)[0]
2192
+ root_vofs = a + vofs
2193
+ block_size = struct.unpack_from('<I', rec, a + vofs + 8)[0]
2194
+
2195
+ alloc = ia_runs = None
2196
+
2197
+ def _index_alloc():
2198
+ nonlocal alloc, ia_runs
2199
+ if alloc is None:
2200
+ alloc = self._attr_value(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
2201
+ _, ia_runs = self._stream_runs(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
2202
+ return alloc, ia_runs
2203
+
2204
+ match = lambda nm: self.names_equal(nm, name)
2205
+ ent = self._find_entry(rec, root_vofs + 16, match)
2206
+ if ent:
2207
+ pos, length, flags = ent
2208
+ if not flags & 1: # leaf entry in the root
2209
+ self._splice_at(rec, root_vofs + 16, pos, length)
2210
+ self.write_record(dir_no, rec)
2211
+ else: # internal entry in the root
2212
+ al, runs = _index_alloc()
2213
+ self._remove_internal(rec, root_vofs + 16, pos, length,
2214
+ lambda: self.write_record(dir_no, rec),
2215
+ runs, block_size, al)
2216
+ return True
2217
+
2218
+ try:
2219
+ al, runs = _index_alloc()
2220
+ except NtfsError:
2221
+ raise NtfsError(errno.ENOENT, f'{name!r} not found in directory {dir_no}')
2222
+ for i in range(len(al) // block_size):
2223
+ blk = bytearray(al[i * block_size:(i + 1) * block_size])
2224
+ if blk[:4] != b'INDX' or not _apply_fixups(blk):
2225
+ continue
2226
+ ent = self._find_entry(blk, 24, match)
2227
+ if not ent:
2228
+ continue
2229
+ pos, length, flags = ent
2230
+ if not flags & 1: # leaf entry in an INDX block
2231
+ self._splice_at(blk, 24, pos, length)
2232
+ self._write_index_block(runs, i, block_size, blk)
2233
+ else: # internal entry in an INDX block
2234
+ def _write(bi=i, bb=blk):
2235
+ self._write_index_block(runs, bi, block_size, bb)
2236
+ self._remove_internal(blk, 24, pos, length, _write,
2237
+ runs, block_size, al)
2238
+ return True
2239
+ raise NtfsError(errno.ENOENT, f'{name!r} not found in directory {dir_no}')
2240
+
2241
+ def remove_dirent(self, path: str, name: str, really: bool) -> dict:
2242
+ info = super().remove_dirent(path, name, really=False) # name, record, seq
2243
+ if not really:
2244
+ return info
2245
+ self.remove_index_entry(self._resolve(path), name)
2246
+ info['removed'] = True
2247
+ return info
2248
+
2249
+ # -- phase 4: index-entry insertion (leaf insert + root grow; split deferred) --
2250
+
2251
+ @staticmethod
2252
+ def _fn_key_name(fn_bytes) -> bytes:
2253
+ return bytes(fn_bytes[66:66 + 2 * fn_bytes[64]])
2254
+
2255
+ def _upcase_seq(self, name_u16: bytes):
2256
+ '''UTF-16LE name upcased through $UpCase as a tuple of code units —
2257
+ the collation key. memoryview.cast avoids struct.unpack, and the
2258
+ full 65536-entry table needs no per-unit bounds check.'''
2259
+ vals = memoryview(name_u16).cast('H')
2260
+ up = self._upcase_table()
2261
+ if up is None:
2262
+ return tuple(vals)
2263
+ if len(up) >= 0x10000:
2264
+ return tuple([up[v] for v in vals])
2265
+ n = len(up)
2266
+ return tuple([up[v] if v < n else v for v in vals])
2267
+
2268
+ @staticmethod
2269
+ def _build_leaf_entry(mref: int, fn_bytes: bytes) -> bytes:
2270
+ klen = len(fn_bytes)
2271
+ elen = (16 + klen + 7) & ~7
2272
+ e = bytearray(elen)
2273
+ struct.pack_into('<QHHH', e, 0, mref, elen, klen, 0) # ref,len,keylen,flags
2274
+ e[16:16 + klen] = fn_bytes
2275
+ return bytes(e)
2276
+
2277
+ @property
2278
+ def _i30(self) -> _IndexSchema:
2279
+ '''The $I30 directory-index schema (COLLATION_FILE_NAME): the stored key
2280
+ is the whole $FILE_NAME, ordered by its upcased name; the entry value
2281
+ is the 8-byte MFT reference.'''
2282
+ s = self.__dict__.get('_i30_cache')
2283
+ if s is None:
2284
+ s = self.__dict__['_i30_cache'] = _IndexSchema(
2285
+ name=self.I30, collation=0x01,
2286
+ sort_key=lambda fn: self._upcase_seq(fn[66:66 + 2 * fn[64]]),
2287
+ build_leaf=self._build_leaf_entry)
2288
+ return s
2289
+
2290
+ @staticmethod
2291
+ def _build_view_leaf(data: bytes, key: bytes, magic: bool = False) -> bytes:
2292
+ '''A leaf INDEX_ENTRY for a $Secure view index: {data_offset, data_length}
2293
+ header, the key at offset 16, then the 20-byte SDS locator as the
2294
+ entry data. $SDH carries a 4-byte "II" (UTF-16LE) magic after the
2295
+ data; $SII does not.'''
2296
+ klen = len(key)
2297
+ doff = 16 + klen
2298
+ tail = b'\x49\x00\x49\x00' if magic else b''
2299
+ elen = (doff + len(data) + len(tail) + 7) & ~7
2300
+ e = bytearray(elen)
2301
+ struct.pack_into('<HH', e, 0, doff, len(data)) # data_offset, data_length
2302
+ struct.pack_into('<HHH', e, 8, elen, klen, 0) # entry_len, key_len, flags
2303
+ e[16:16 + klen] = key
2304
+ e[doff:doff + len(data)] = data
2305
+ e[doff + len(data):doff + len(data) + len(tail)] = tail
2306
+ return bytes(e)
2307
+
2308
+ @property
2309
+ def _sii(self) -> _IndexSchema:
2310
+ '''$Secure $SII: key = security_id (u32), COLLATION_NTOFS_ULONG.'''
2311
+ s = self.__dict__.get('_sii_cache')
2312
+ if s is None:
2313
+ s = self.__dict__['_sii_cache'] = _IndexSchema(
2314
+ name='$SII'.encode('utf-16-le'), collation=0x10,
2315
+ sort_key=lambda k: int.from_bytes(k[:4], 'little'),
2316
+ build_leaf=lambda data, key: self._build_view_leaf(data, key))
2317
+ return s
2318
+
2319
+ @property
2320
+ def _sdh(self) -> _IndexSchema:
2321
+ '''$Secure $SDH: key = {hash u32, id u32}, COLLATION_NTOFS_SECURITY_HASH.'''
2322
+ s = self.__dict__.get('_sdh_cache')
2323
+ if s is None:
2324
+ s = self.__dict__['_sdh_cache'] = _IndexSchema(
2325
+ name='$SDH'.encode('utf-16-le'), collation=0x12,
2326
+ sort_key=lambda k: struct.unpack_from('<II', k),
2327
+ build_leaf=lambda data, key: self._build_view_leaf(data, key, magic=True))
2328
+ return s
2329
+
2330
+ @property
2331
+ def _ix(self) -> _IndexSchema:
2332
+ '''The index schema the B+ write engine currently operates on — $I30 by
2333
+ default. (A generic view-index insert will scope self._ix_override.)'''
2334
+ return getattr(self, '_ix_override', None) or self._i30
2335
+
2336
+ def _resident_grow_root(self, rec: bytearray, root_attr_off: int,
2337
+ delta: int) -> None:
2338
+ '''Resize the resident $INDEX_ROOT attribute by delta bytes in place
2339
+ (negative shrinks), shifting the attributes after it and the
2340
+ record's used size. Refuses if the record has no room for a grow
2341
+ (that needs small→large conversion).'''
2342
+ in_use = struct.unpack_from('<I', rec, 0x18)[0]
2343
+ if in_use + delta > len(rec):
2344
+ raise NtfsError(errno.ENOTSUP,
2345
+ 'directory record full — cannot grow the resident index root')
2346
+ attr_len = struct.unpack_from('<I', rec, root_attr_off + 4)[0]
2347
+ tail = bytes(rec[root_attr_off + attr_len:in_use])
2348
+ rec[root_attr_off + attr_len + delta:
2349
+ root_attr_off + attr_len + delta + len(tail)] = tail
2350
+ for i in range(root_attr_off + attr_len, root_attr_off + attr_len + delta):
2351
+ rec[i] = 0 # grow: zero the inserted gap
2352
+ for i in range(in_use + delta, in_use):
2353
+ rec[i] = 0 # shrink: zero the freed tail
2354
+ vlen = struct.unpack_from('<I', rec, root_attr_off + 0x10)[0]
2355
+ vofs = struct.unpack_from('<H', rec, root_attr_off + 0x14)[0]
2356
+ struct.pack_into('<I', rec, root_attr_off + 4, attr_len + delta)
2357
+ struct.pack_into('<I', rec, root_attr_off + 0x10, vlen + delta)
2358
+ struct.pack_into('<I', rec, 0x18, in_use + delta)
2359
+ hdr = root_attr_off + vofs + 16 # INDEX_HEADER.allocated_size
2360
+ asz = struct.unpack_from('<I', rec, hdr + 8)[0]
2361
+ struct.pack_into('<I', rec, hdr + 8, asz + delta)
2362
+
2363
+ # -- phase 4d: attribute creation + small→large conversion --
2364
+
2365
+ def _add_attr(self, rec: bytearray, attr_type: int, name_u16: bytes,
2366
+ resident: bool, value: bytes = b'', runs=None,
2367
+ data_size: int = 0) -> int:
2368
+ '''Build an attribute and insert it into the base record in (type, name)
2369
+ order. resident: uses `value`. non-resident: uses `runs` (encoded as
2370
+ mapping pairs) + data_size. Returns the attribute's record offset;
2371
+ raises ENOTSUP if the record is full ($ATTRIBUTE_LIST spill needed).'''
2372
+ name_len = len(name_u16) // 2
2373
+ if resident:
2374
+ name_off = 0x18
2375
+ val_off = ((name_off + len(name_u16) + 7) & ~7) if name_len else 0x18
2376
+ alen = (val_off + len(value) + 7) & ~7
2377
+ attr = bytearray(alen)
2378
+ struct.pack_into('<IIBBH', attr, 0, attr_type, alen, 0, name_len,
2379
+ name_off if name_len else 0)
2380
+ struct.pack_into('<IH', attr, 0x10, len(value), val_off)
2381
+ if name_len:
2382
+ attr[name_off:name_off + len(name_u16)] = name_u16
2383
+ attr[val_off:val_off + len(value)] = value
2384
+ else:
2385
+ name_off = 0x40
2386
+ mp_off = ((name_off + len(name_u16) + 7) & ~7) if name_len else 0x40
2387
+ mp = _encode_mapping_pairs(runs)
2388
+ alen = (mp_off + len(mp) + 7) & ~7
2389
+ total = sum(n for _lcn, n in runs)
2390
+ attr = bytearray(alen)
2391
+ struct.pack_into('<IIBBH', attr, 0, attr_type, alen, 1, name_len,
2392
+ name_off if name_len else 0)
2393
+ struct.pack_into('<qq', attr, 0x10, 0, total - 1) # lowest/highest vcn
2394
+ struct.pack_into('<H', attr, 0x20, mp_off)
2395
+ struct.pack_into('<qqq', attr, 0x28, data_size, data_size, data_size)
2396
+ if name_len:
2397
+ attr[name_off:name_off + len(name_u16)] = name_u16
2398
+ attr[mp_off:mp_off + len(mp)] = mp
2399
+ inst = struct.unpack_from('<H', rec, 0x28)[0]
2400
+ struct.pack_into('<H', attr, 0x0E, inst)
2401
+ struct.pack_into('<H', rec, 0x28, inst + 1)
2402
+
2403
+ in_use = struct.unpack_from('<I', rec, 0x18)[0]
2404
+ if in_use + alen > len(rec):
2405
+ raise NtfsError(errno.ENOTSUP, 'record full — $ATTRIBUTE_LIST spill needed')
2406
+ a = struct.unpack_from('<H', rec, 0x14)[0]
2407
+ while a + 4 <= in_use: # (type, name) order
2408
+ t = struct.unpack_from('<I', rec, a)[0]
2409
+ if t == 0xFFFFFFFF or t > attr_type:
2410
+ break
2411
+ a += struct.unpack_from('<I', rec, a + 4)[0]
2412
+ tail = bytes(rec[a:in_use])
2413
+ rec[a + alen:a + alen + len(tail)] = tail
2414
+ rec[a:a + alen] = attr
2415
+ struct.pack_into('<I', rec, 0x18, in_use + alen)
2416
+ return a
2417
+
2418
+ def _replace_resident_value(self, rec: bytearray, attr_off: int,
2419
+ new_value: bytes) -> None:
2420
+ '''Replace a resident attribute's value (any length), shifting the
2421
+ attributes after it and the record's used size. Shrinking frees
2422
+ record space; growing needs room (else the caller must spill).'''
2423
+ vofs = struct.unpack_from('<H', rec, attr_off + 0x14)[0]
2424
+ old_alen = struct.unpack_from('<I', rec, attr_off + 4)[0]
2425
+ new_alen = (vofs + len(new_value) + 7) & ~7
2426
+ in_use = struct.unpack_from('<I', rec, 0x18)[0]
2427
+ if in_use + (new_alen - old_alen) > len(rec):
2428
+ raise NtfsError(errno.ENOTSUP, 'record full')
2429
+ tail = bytes(rec[attr_off + old_alen:in_use])
2430
+ rec[attr_off + new_alen:attr_off + new_alen + len(tail)] = tail
2431
+ rec[attr_off + vofs:attr_off + vofs + len(new_value)] = new_value
2432
+ struct.pack_into('<I', rec, attr_off + 4, new_alen)
2433
+ struct.pack_into('<I', rec, attr_off + 0x10, len(new_value))
2434
+ new_in_use = in_use + (new_alen - old_alen)
2435
+ struct.pack_into('<I', rec, 0x18, new_in_use)
2436
+ for i in range(new_in_use, in_use): # zero any freed tail
2437
+ rec[i] = 0
2438
+
2439
+ def _small_to_large(self, dir_no: int, block_size: int, vpb: int) -> None:
2440
+ '''Convert a SMALL_INDEX (resident $INDEX_ROOT only) into a LARGE index:
2441
+ shrink the root to a single END→block pointer (freeing record space),
2442
+ create the $INDEX_ALLOCATION and $BITMAP attributes in that space,
2443
+ and move the root's entries into the new block.'''
2444
+ rec = self._load_record(dir_no)
2445
+ if rec is None:
2446
+ raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
2447
+ ra, _l = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
2448
+ rvofs = ra + struct.unpack_from('<H', rec, ra + 0x14)[0]
2449
+ reals, end = self._decode_node(rec, rvofs + 16) # leaf entries + END
2450
+ # 1) shrink the root to a minimal LARGE root: prefix + header + END→0
2451
+ prefix = bytes(rec[rvofs:rvofs + 16])
2452
+ end_node = self._end_node(0)
2453
+ vlen = 32 + len(end_node)
2454
+ ih = struct.pack('<IIIBxxx', 16, 16 + len(end_node), vlen - 16, 1)
2455
+ self._replace_resident_value(rec, ra, prefix + ih + end_node)
2456
+ # 2) create $INDEX_ALLOCATION (1 block) and $BITMAP in the freed space
2457
+ cpb = block_size // self._cluster_size
2458
+ runs = self.alloc_clusters(cpb)
2459
+ try:
2460
+ self._add_attr(rec, AT_INDEX_ALLOCATION, self._ix.name, resident=False,
2461
+ runs=[(l, n) for l, n in runs], data_size=block_size)
2462
+ bm = bytearray(8) # 1 bit/block; 8 = 64 blocks
2463
+ bm[0] = 1 # block 0 in use
2464
+ self._add_attr(rec, AT_BITMAP, self._ix.name, resident=True, value=bytes(bm))
2465
+ except NtfsError:
2466
+ self.free_clusters(runs)
2467
+ raise
2468
+ self.write_record(dir_no, rec)
2469
+ if getattr(self, '_ia_cache', None):
2470
+ self._ia_cache.pop(dir_no, None)
2471
+ # 3) move the old root entries into block 0
2472
+ blk = self._new_indx_block(0, block_size)
2473
+ self._pack_node(blk, 24, reals, end)
2474
+ self._write_block(dir_no, 0, block_size, blk)
2475
+
2476
+ # -- phase 4b: node split — insertion never refuses (large indexes) --
2477
+
2478
+ @staticmethod
2479
+ def _leaf_end() -> bytes:
2480
+ return struct.pack('<QHHHH', 0, 16, 0, 2, 0) # END, no subnode
2481
+
2482
+ @staticmethod
2483
+ def _end_node(vcn: int) -> bytes:
2484
+ return struct.pack('<QHHHH', 0, 24, 0, 3, 0) + struct.pack('<q', vcn)
2485
+
2486
+ @staticmethod
2487
+ def _entry_subnode(e: bytes) -> int:
2488
+ return struct.unpack_from('<q', e, len(e) - 8)[0]
2489
+
2490
+ @staticmethod
2491
+ def _set_subnode(e: bytes, vcn: int) -> bytes:
2492
+ b = bytearray(e)
2493
+ struct.pack_into('<q', b, len(b) - 8, vcn)
2494
+ return bytes(b)
2495
+
2496
+ @staticmethod
2497
+ def _make_node_entry(e: bytes, vcn: int) -> bytes:
2498
+ '''Promote entry e to a NODE separator: the entry followed by the
2499
+ 8-byte subnode VCN. A $Secure view-index entry carries its 20-byte
2500
+ data (+ $SDH magic) into the separator — NTFS keeps the median's data
2501
+ in the internal node — while a directory ($I30) entry is just
2502
+ header+key+pad, so this is byte-identical to the old header+key form
2503
+ there. An entry that is already a separator (multi-level split)
2504
+ must first shed its own subnode VCN, or the ghost 8 bytes make
2505
+ entry_length non-canonical — chkdsk flags that as an index error.'''
2506
+ elen = struct.unpack_from('<H', e, 8)[0]
2507
+ flags = struct.unpack_from('<H', e, 12)[0]
2508
+ if flags & 1:
2509
+ elen -= 8
2510
+ base = bytes(e[:elen])
2511
+ base += b'\x00' * (-len(base) % 8)
2512
+ b = bytearray(base + struct.pack('<q', vcn))
2513
+ struct.pack_into('<H', b, 8, len(b))
2514
+ struct.pack_into('<H', b, 12, 1) # NODE, not END
2515
+ return bytes(b)
2516
+
2517
+ @staticmethod
2518
+ def _decode_node(buf, hdr_off):
2519
+ eo, il = struct.unpack_from('<II', buf, hdr_off)
2520
+ pos, limit, reals, end = hdr_off + eo, hdr_off + il, [], None
2521
+ while pos + 16 <= limit:
2522
+ length = struct.unpack_from('<H', buf, pos + 8)[0]
2523
+ flags = struct.unpack_from('<H', buf, pos + 12)[0]
2524
+ if length < 16:
2525
+ break
2526
+ e = bytes(buf[pos:pos + length])
2527
+ if flags & 2:
2528
+ end = e
2529
+ break
2530
+ reals.append(e)
2531
+ pos += length
2532
+ if end is None:
2533
+ raise NtfsError(errno.EIO, 'index node without END entry')
2534
+ return reals, end
2535
+
2536
+ @staticmethod
2537
+ def _pack_node(buf, hdr_off, reals, end) -> bool:
2538
+ '''Write reals+end into the node; False if it overflows allocated_size.
2539
+ Also fixes the INDEX_HEADER node flag from the END entry.'''
2540
+ eo = struct.unpack_from('<I', buf, hdr_off)[0]
2541
+ asz = struct.unpack_from('<I', buf, hdr_off + 8)[0]
2542
+ old_il = struct.unpack_from('<I', buf, hdr_off + 4)[0]
2543
+ body = b''.join(reals) + end
2544
+ new_il = eo + len(body)
2545
+ if new_il > asz:
2546
+ return False
2547
+ p = hdr_off + eo
2548
+ buf[p:p + len(body)] = body
2549
+ for i in range(p + len(body), hdr_off + max(old_il, new_il)):
2550
+ buf[i] = 0
2551
+ struct.pack_into('<I', buf, hdr_off + 4, new_il)
2552
+ node_flag = struct.unpack_from('<H', end, 12)[0] & 1
2553
+ buf[hdr_off + 12] = node_flag # INDEX_HEADER.flags: has children?
2554
+ return True
2555
+
2556
+ def _new_indx_block(self, vcn: int, block_size: int):
2557
+ blk = bytearray(block_size)
2558
+ blk[:4] = b'INDX'
2559
+ usa_ofs, usa_count = 40, 1 + block_size // 512
2560
+ struct.pack_into('<HH', blk, 4, usa_ofs, usa_count)
2561
+ struct.pack_into('<q', blk, 16, vcn)
2562
+ entries_off = ((usa_ofs + 2 * usa_count + 7) & ~7) - 24
2563
+ struct.pack_into('<IIIB', blk, 24, entries_off, entries_off,
2564
+ block_size - 24, 0)
2565
+ return blk
2566
+
2567
+ def _ia(self, dir_no):
2568
+ '''Cached (size, runs) of $INDEX_ALLOCATION — invalidated only on grow.'''
2569
+ cache = getattr(self, '_ia_cache', None)
2570
+ if cache is None:
2571
+ cache = self._ia_cache = {}
2572
+ got = cache.get(dir_no)
2573
+ if got is None:
2574
+ got = cache[dir_no] = self._stream_runs(dir_no, AT_INDEX_ALLOCATION,
2575
+ self._ix.name)
2576
+ return got
2577
+
2578
+ def _read_block(self, dir_no, block_index, block_size):
2579
+ _size, runs = self._ia(dir_no)
2580
+ return bytearray(self._runs_read(runs, block_index * block_size, block_size))
2581
+
2582
+ def _write_block(self, dir_no, block_index, block_size, blk) -> None:
2583
+ _s, runs = self._ia(dir_no)
2584
+ self._write_index_block(runs, block_index, block_size, blk)
2585
+
2586
+ def _alloc_index_block(self, dir_no, block_size, vpb):
2587
+ '''(block_index, vcn) of a free INDX block — reusing a clear $BITMAP
2588
+ bit, else growing $INDEX_ALLOCATION by one block.'''
2589
+ bm = self._read_i30_bitmap(dir_no)
2590
+ size, _runs = self._ia(dir_no)
2591
+ n_blocks = size // block_size
2592
+ for i in range(n_blocks):
2593
+ if not bm[i >> 3] & (1 << (i & 7)):
2594
+ bm[i >> 3] |= 1 << (i & 7)
2595
+ self._write_i30_bitmap(dir_no, bm)
2596
+ return i, i * vpb
2597
+ return self._grow_index_alloc(dir_no, block_size, vpb, bm, n_blocks)
2598
+
2599
+ def _base_attr(self, rec, atype, name):
2600
+ for a, t, ln in _attrs(rec):
2601
+ if t == atype:
2602
+ nlen = rec[a + 9]
2603
+ nofs = struct.unpack_from('<H', rec, a + 10)[0]
2604
+ aname = bytes(rec[a + nofs:a + nofs + 2 * nlen]) if nlen else b''
2605
+ if aname == (name or b''):
2606
+ return a, ln
2607
+ return None
2608
+
2609
+ def _read_i30_bitmap(self, dir_no):
2610
+ found = self._find_attr(dir_no, AT_BITMAP, self._ix.name) # base or extension
2611
+ if not found:
2612
+ raise NtfsError(errno.ENOENT, 'directory has no $I30 $BITMAP')
2613
+ _no, rec, a = found
2614
+ ln = struct.unpack_from('<I', rec, a + 4)[0]
2615
+ if rec[a + 8] == 0:
2616
+ vo = struct.unpack_from('<H', rec, a + 20)[0]
2617
+ vl = struct.unpack_from('<I', rec, a + 16)[0]
2618
+ return bytearray(rec[a + vo:a + vo + vl])
2619
+ _p, _ph, runs = _check_mapping_pairs(bytes(rec), a, ln, self.nr_clusters())
2620
+ return bytearray(self._runs_read(runs, 0, struct.unpack_from('<q', rec, a + 48)[0]))
2621
+
2622
+ def _write_i30_bitmap(self, dir_no, bm):
2623
+ rec_no, rec, a = self._find_attr(dir_no, AT_BITMAP, self._ix.name)
2624
+ ln = struct.unpack_from('<I', rec, a + 4)[0]
2625
+ if rec[a + 8] == 0:
2626
+ vo = struct.unpack_from('<H', rec, a + 20)[0]
2627
+ vl = struct.unpack_from('<I', rec, a + 16)[0]
2628
+ rec[a + vo:a + vo + vl] = bytes(bm)[:vl]
2629
+ self.write_record(rec_no, rec)
2630
+ else:
2631
+ _p, _ph, runs = _check_mapping_pairs(bytes(rec), a, ln, self.nr_clusters())
2632
+ self._runs_write(runs, 0, bytes(bm))
2633
+
2634
+ def _grow_i30_bitmap(self, dir_no, want_bytes: int):
2635
+ '''Extend the $I30 $BITMAP to at least want_bytes. If its record is full,
2636
+ spill $INDEX_ALLOCATION + $BITMAP to an extension record (only when
2637
+ the bitmap still lives in the base) and retry there.'''
2638
+ rec_no, rec, a = self._find_attr(dir_no, AT_BITMAP, self._ix.name)
2639
+ if rec[a + 8] != 0:
2640
+ raise NtfsError(errno.ENOTSUP, 'non-resident $I30 $BITMAP growth not implemented')
2641
+ vo = struct.unpack_from('<H', rec, a + 0x14)[0]
2642
+ vl = struct.unpack_from('<I', rec, a + 0x10)[0]
2643
+ if want_bytes <= vl:
2644
+ return
2645
+ cur = bytes(rec[a + vo:a + vo + vl])
2646
+ try:
2647
+ self._replace_resident_value(rec, a, cur + b'\x00' * (want_bytes - vl))
2648
+ except NtfsError: # record full
2649
+ if rec_no != dir_no:
2650
+ raise NtfsError(errno.ENOTSUP, 'extension $BITMAP record full — '
2651
+ 'further spill not implemented')
2652
+ self._spill_index_alloc(dir_no) # frees the base (or re-raises)
2653
+ return self._grow_i30_bitmap(dir_no, want_bytes)
2654
+ self.write_record(rec_no, rec)
2655
+
2656
+ # -- phase 4e: $ATTRIBUTE_LIST spill (base record full → extension records) --
2657
+
2658
+ def _find_attr(self, dir_no, attr_type, name):
2659
+ '''(record_no, fixed-up record, attr_off) of an attribute — searching the
2660
+ base record, then extension records via $ATTRIBUTE_LIST. None if
2661
+ absent. name is UTF-16LE (or b'' for unnamed).'''
2662
+ base = self._load_record(dir_no)
2663
+ if base is None:
2664
+ raise NtfsError(errno.EIO, f'record {dir_no} unreadable')
2665
+ loc = self._base_attr(base, attr_type, name)
2666
+ if loc:
2667
+ return dir_no, base, loc[0]
2668
+ for ext_no in self._extension_records(dir_no):
2669
+ ext = self._load_record(ext_no)
2670
+ if ext is None:
2671
+ continue
2672
+ loc = self._base_attr(ext, attr_type, name)
2673
+ if loc:
2674
+ return ext_no, ext, loc[0]
2675
+ return None
2676
+
2677
+ def _build_extension_record(self, base_no, base_seq, ext_seq, attrs):
2678
+ '''A minimal extension FILE record holding `attrs` (each re-instanced),
2679
+ with base_reference pointing back at the base. Returns the record.'''
2680
+ rs = self._rec_size
2681
+ rec = bytearray(rs)
2682
+ rec[:4] = b'FILE'
2683
+ usa_ofs, usa_count = 0x30, 1 + rs // 512
2684
+ struct.pack_into('<HH', rec, 4, usa_ofs, usa_count)
2685
+ struct.pack_into('<H', rec, 0x10, ext_seq) # sequence
2686
+ struct.pack_into('<H', rec, 0x12, 0) # hard_link_count
2687
+ attrs_off = (usa_ofs + 2 * usa_count + 7) & ~7
2688
+ struct.pack_into('<H', rec, 0x14, attrs_off) # attrs offset
2689
+ struct.pack_into('<H', rec, 0x16, 1) # flags: in use
2690
+ struct.pack_into('<Q', rec, 0x20, base_no | (base_seq << 48))
2691
+ off = attrs_off
2692
+ for inst, attr in enumerate(attrs):
2693
+ a = bytearray(attr)
2694
+ struct.pack_into('<H', a, 0x0E, inst) # instance id
2695
+ rec[off:off + len(a)] = a
2696
+ off += len(a)
2697
+ struct.pack_into('<I', rec, off, 0xFFFFFFFF)
2698
+ struct.pack_into('<I', rec, 0x18, off + 4) # bytes_in_use
2699
+ struct.pack_into('<I', rec, 0x1C, rs) # bytes_allocated
2700
+ struct.pack_into('<H', rec, 0x28, len(attrs)) # next_attr_id
2701
+ struct.pack_into('<I', rec, 0x2C, 0) # record number (unused)
2702
+ return rec
2703
+
2704
+ @staticmethod
2705
+ def _al_entry(atype, name_u16, vcn, ref_no, ref_seq, inst):
2706
+ nl = len(name_u16) // 2
2707
+ rlen = (0x1A + len(name_u16) + 7) & ~7
2708
+ e = bytearray(rlen)
2709
+ struct.pack_into('<IHBB', e, 0, atype, rlen, nl, 0x1A if nl else 0)
2710
+ struct.pack_into('<Q', e, 8, vcn)
2711
+ struct.pack_into('<Q', e, 16, ref_no | (ref_seq << 48))
2712
+ struct.pack_into('<H', e, 24, inst)
2713
+ if nl:
2714
+ e[0x1A:0x1A + len(name_u16)] = name_u16
2715
+ return atype, name_u16, bytes(e)
2716
+
2717
+ def _spill_index_alloc(self, dir_no):
2718
+ '''Move $INDEX_ALLOCATION and $BITMAP out of a full base record into a
2719
+ fresh extension record, adding an $ATTRIBUTE_LIST to the base so both
2720
+ can keep growing in the roomy extension. First spill only (a base
2721
+ that already has a list is refused). Atomic: the extension
2722
+ record is freed if anything fails.'''
2723
+ base = self._load_record(dir_no)
2724
+ if base is None:
2725
+ raise NtfsError(errno.EIO, f'record {dir_no} unreadable')
2726
+ if self._base_attr(base, AT_ATTRIBUTE_LIST, b''):
2727
+ raise NtfsError(errno.ENOTSUP, 'directory already has an '
2728
+ '$ATTRIBUTE_LIST — further spill not implemented')
2729
+ base_seq = struct.unpack_from('<H', base, 0x10)[0]
2730
+ # attributes to relocate, by record offset (removed high→low)
2731
+ move = []
2732
+ for atype in (AT_INDEX_ALLOCATION, AT_BITMAP):
2733
+ loc = self._base_attr(base, atype, self._ix.name)
2734
+ if loc:
2735
+ a = loc[0]
2736
+ alen = struct.unpack_from('<I', base, a + 4)[0]
2737
+ move.append((a, atype, bytes(base[a:a + alen])))
2738
+ if not any(m[1] == AT_INDEX_ALLOCATION for m in move):
2739
+ raise NtfsError(errno.ENOTSUP, '$INDEX_ALLOCATION not in base record')
2740
+
2741
+ ext_no = self.alloc_record()
2742
+ try:
2743
+ old = self.read_record(ext_no)
2744
+ ext_seq = 1
2745
+ if old and old[:4] == b'FILE':
2746
+ ext_seq = (struct.unpack_from('<H', old, 0x10)[0] + 1) & 0xFFFF or 1
2747
+ moved_bytes = [b for _a, _t, b in move]
2748
+ ext = self._build_extension_record(dir_no, base_seq, ext_seq, moved_bytes)
2749
+ ext_inst = {t: i for i, (_a, t, _b) in enumerate(move)} # instance in ext
2750
+
2751
+ # remove the moved attributes from the base (high offset first)
2752
+ for a, _t, blob in sorted(move, key=lambda m: -m[0]):
2753
+ in_use = struct.unpack_from('<I', base, 0x18)[0]
2754
+ alen = len(blob)
2755
+ tail = bytes(base[a + alen:in_use])
2756
+ base[a:a + len(tail)] = tail
2757
+ struct.pack_into('<I', base, 0x18, in_use - alen)
2758
+ for i in range(in_use - alen, in_use):
2759
+ base[i] = 0
2760
+
2761
+ # $ATTRIBUTE_LIST: remaining base attrs → base, moved → extension
2762
+ entries, off = [], struct.unpack_from('<H', base, 0x14)[0]
2763
+ end = struct.unpack_from('<I', base, 0x18)[0]
2764
+ while off + 4 <= end:
2765
+ t = struct.unpack_from('<I', base, off)[0]
2766
+ if t == 0xFFFFFFFF:
2767
+ break
2768
+ ln = struct.unpack_from('<I', base, off + 4)[0]
2769
+ nl = base[off + 9]
2770
+ nofs = struct.unpack_from('<H', base, off + 10)[0]
2771
+ nm = bytes(base[off + nofs:off + nofs + 2 * nl]) if nl else b''
2772
+ inst = struct.unpack_from('<H', base, off + 0x0E)[0]
2773
+ entries.append(self._al_entry(t, nm, 0, dir_no, base_seq, inst))
2774
+ off += ln
2775
+ for _a, t, _b in move:
2776
+ entries.append(self._al_entry(t, self._ix.name, 0, ext_no, ext_seq,
2777
+ ext_inst[t]))
2778
+ entries.sort(key=lambda x: (x[0], x[1]))
2779
+ al_value = b''.join(e for _t, _n, e in entries)
2780
+ self._add_attr(base, AT_ATTRIBUTE_LIST, b'', resident=True, value=al_value)
2781
+
2782
+ self.write_record(ext_no, ext)
2783
+ self.write_record(dir_no, base)
2784
+ except NtfsError:
2785
+ self.free_record(ext_no) # undo the extension allocation
2786
+ raise
2787
+ if getattr(self, '_ia_cache', None):
2788
+ self._ia_cache.pop(dir_no, None)
2789
+ if getattr(self, '_attr_memo', None):
2790
+ self._attr_memo.clear()
2791
+
2792
+ def _grow_index_alloc(self, dir_no, block_size, vpb, bm, n_blocks):
2793
+ '''Append one INDX block to $INDEX_ALLOCATION and set its $BITMAP bit,
2794
+ growing the resident $BITMAP if needed. When the $INDEX_ALLOCATION
2795
+ mapping pairs would overflow the base record, spill it to an
2796
+ extension record (via $ATTRIBUTE_LIST) and retry.'''
2797
+ if (n_blocks >> 3) >= len(bm):
2798
+ self._grow_i30_bitmap(dir_no, (n_blocks // 8 + 8) & ~7)
2799
+ bm = self._read_i30_bitmap(dir_no)
2800
+ cpb = block_size // self._cluster_size
2801
+ found = self._find_attr(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
2802
+ if not found:
2803
+ raise NtfsError(errno.EIO, '$INDEX_ALLOCATION not found')
2804
+ ia_no, rec, a = found
2805
+ ln = struct.unpack_from('<I', rec, a + 4)[0]
2806
+ _p, _ph, existing = _check_mapping_pairs(bytes(rec), a, ln, self.nr_clusters())
2807
+ merged = [(l, n) for l, n, _v in existing]
2808
+ hint = merged[-1][0] + merged[-1][1] if merged else 0
2809
+ new_runs = self.alloc_clusters(cpb, near_lcn=hint)
2810
+ for lcn, rl in new_runs:
2811
+ if merged and merged[-1][0] + merged[-1][1] == lcn:
2812
+ merged[-1] = (merged[-1][0], merged[-1][1] + rl)
2813
+ else:
2814
+ merged.append((lcn, rl))
2815
+ mp = _encode_mapping_pairs(merged)
2816
+ mp_off = struct.unpack_from('<H', rec, a + 32)[0]
2817
+ if mp_off + len(mp) > ln:
2818
+ # the attribute is sized tightly, so a runlist that fragments
2819
+ # routinely needs a few more bytes: grow it in place while its
2820
+ # record has room, spilling only when the record is truly full
2821
+ delta = (mp_off + len(mp) - ln + 7) & ~7
2822
+ in_use = struct.unpack_from('<I', rec, 0x18)[0]
2823
+ if in_use + delta <= len(rec):
2824
+ tail = bytes(rec[a + ln:in_use])
2825
+ rec[a + ln + delta:a + ln + delta + len(tail)] = tail
2826
+ for i in range(a + ln, a + ln + delta):
2827
+ rec[i] = 0
2828
+ struct.pack_into('<I', rec, a + 4, ln + delta)
2829
+ struct.pack_into('<I', rec, 0x18, in_use + delta)
2830
+ ln += delta
2831
+ elif ia_no == dir_no: # still in the base — spill and retry
2832
+ self.free_clusters(new_runs)
2833
+ self._spill_index_alloc(dir_no)
2834
+ return self._grow_index_alloc(dir_no, block_size, vpb,
2835
+ self._read_i30_bitmap(dir_no), n_blocks)
2836
+ else:
2837
+ self.free_clusters(new_runs)
2838
+ raise NtfsError(errno.ENOTSUP, 'extension $INDEX_ALLOCATION record '
2839
+ 'full — further spill not implemented')
2840
+ for i in range(a + mp_off, a + ln):
2841
+ rec[i] = 0
2842
+ rec[a + mp_off:a + mp_off + len(mp)] = mp
2843
+ new_size = (n_blocks + 1) * block_size
2844
+ struct.pack_into('<q', rec, a + 24, (n_blocks + 1) * vpb - 1) # highest_vcn
2845
+ struct.pack_into('<q', rec, a + 40, new_size) # allocated
2846
+ struct.pack_into('<q', rec, a + 48, new_size) # data
2847
+ struct.pack_into('<q', rec, a + 56, new_size) # initialized
2848
+ self.write_record(ia_no, rec) # base or extension
2849
+ bm[n_blocks >> 3] |= 1 << (n_blocks & 7)
2850
+ self._write_i30_bitmap(dir_no, bm)
2851
+ getattr(self, '_ia_cache', {}).pop(dir_no, None)
2852
+ return n_blocks, n_blocks * vpb
2853
+
2854
+ def _node_buf(self, dir_no, ref, block_size):
2855
+ '''(buf, hdr_off, root_attr_off|None) for a node reference.'''
2856
+ if ref[0] == 'root':
2857
+ rec = self._load_record(dir_no)
2858
+ if rec is None:
2859
+ raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
2860
+ a, _ln = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
2861
+ vofs = struct.unpack_from('<H', rec, a + 20)[0]
2862
+ return rec, a + vofs + 16, a
2863
+ i = ref[1]
2864
+ blk = self._read_block(dir_no, i, block_size)
2865
+ if blk[:4] != b'INDX' or not _apply_fixups(blk):
2866
+ raise NtfsError(errno.EIO, f'index block {i} unreadable')
2867
+ return blk, 24, None
2868
+
2869
+ def _node_flush(self, dir_no, ref, buf, block_size):
2870
+ if ref[0] == 'root':
2871
+ self.write_record(dir_no, buf)
2872
+ else:
2873
+ self._write_block(dir_no, ref[1], block_size, buf)
2874
+
2875
+ def _pack_or_grow(self, dir_no, ref, buf, hdr_off, root_attr_off,
2876
+ reals, end, block_size) -> bool:
2877
+ '''Pack and flush; grow a resident root if it overflows. Returns True if
2878
+ it fit (done), False for a block that must split. Root that can't
2879
+ grow → ENOTSUP (small→large / root split not implemented here).'''
2880
+ if self._pack_node(buf, hdr_off, reals, end):
2881
+ self._node_flush(dir_no, ref, buf, block_size)
2882
+ return True
2883
+ if ref[0] != 'root':
2884
+ return False
2885
+ eo = struct.unpack_from('<I', buf, hdr_off)[0]
2886
+ asz = struct.unpack_from('<I', buf, hdr_off + 8)[0]
2887
+ need = eo + len(b''.join(reals) + end)
2888
+ delta = (need - asz + 7) & ~7
2889
+ self._resident_grow_root(buf, root_attr_off, delta) # raises ENOTSUP if full
2890
+ if not self._pack_node(buf, hdr_off, reals, end):
2891
+ raise NtfsError(errno.ENOTSUP, 'root would need a height increase '
2892
+ '(small→large / root split) here — not implemented')
2893
+ self._node_flush(dir_no, ref, buf, block_size)
2894
+ return True
2895
+
2896
+ @staticmethod
2897
+ def _would_pack(hdr_off_asz_eo, reals, end) -> bool:
2898
+ asz, eo = hdr_off_asz_eo
2899
+ return eo + len(b''.join(reals) + end) <= asz
2900
+
2901
+ def insert_index_entry(self, dir_no: int, mref: int, fn_bytes: bytes) -> bool:
2902
+ '''Insert a $FILE_NAME leaf entry into a directory's $I30 in collation
2903
+ order. A full leaf splits and promotes its median to the parent
2904
+ (which grows if it is the resident root). Plan-then-commit and
2905
+ bounded: the whole operation is verified feasible BEFORE any write,
2906
+ and a split that would cascade past the immediate parent, need a root
2907
+ height increase, or overflow the $INDEX_ALLOCATION attribute record
2908
+ raises ENOTSUP with nothing written. EEXIST on a
2909
+ duplicate name.'''
2910
+ rec = self._load_record(dir_no)
2911
+ if rec is None:
2912
+ raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
2913
+ loc = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
2914
+ if not loc:
2915
+ raise NtfsError(errno.ENOENT, f'record {dir_no} has no $I30 index root')
2916
+ a, _ln = loc
2917
+ vofs = struct.unpack_from('<H', rec, a + 20)[0]
2918
+ block_size = struct.unpack_from('<I', rec, a + vofs + 8)[0]
2919
+ vpb = max(block_size // self._cluster_size, 1)
2920
+ new_key = self._ix.sort_key(fn_bytes) # collation key of the new entry
2921
+ entry = self._ix.build_leaf(mref, fn_bytes)
2922
+
2923
+ # -- descend, recording the path root→leaf --
2924
+ path, ref = [], ('root', dir_no)
2925
+ while True:
2926
+ buf, hdr_off, rao = self._node_buf(dir_no, ref, block_size)
2927
+ reals, end = self._decode_node(buf, hdr_off)
2928
+ idx, descend = None, None
2929
+ for k, e in enumerate(reals):
2930
+ ek = self._ix.entry_sort_key(e)
2931
+ if new_key == ek:
2932
+ raise NtfsError(errno.EEXIST, 'name already present in index')
2933
+ if new_key < ek:
2934
+ idx = k
2935
+ if struct.unpack_from('<H', e, 12)[0] & 1:
2936
+ descend = self._entry_subnode(e)
2937
+ break
2938
+ if idx is None:
2939
+ idx = len(reals)
2940
+ if struct.unpack_from('<H', end, 12)[0] & 1:
2941
+ descend = self._entry_subnode(end)
2942
+ asz = struct.unpack_from('<I', buf, hdr_off + 8)[0]
2943
+ eo = struct.unpack_from('<I', buf, hdr_off)[0]
2944
+ path.append({'ref': ref, 'buf': buf, 'hdr': hdr_off, 'rao': rao,
2945
+ 'reals': reals, 'end': end, 'idx': idx, 'cap': (asz, eo)})
2946
+ if descend is None:
2947
+ break
2948
+ ref = ('block', descend // vpb)
2949
+
2950
+ leaf = path[-1]
2951
+ # small (root-only) index whose root can't hold the new entry even
2952
+ # grown → convert to a large index, then retry into the block tree
2953
+ if leaf['ref'][0] == 'root':
2954
+ body_len = len(b''.join(leaf['reals']) + [leaf['end']][0]) \
2955
+ + len(entry)
2956
+ if not self._root_body_room(leaf['buf'], leaf['rao'], body_len):
2957
+ self._small_to_large(dir_no, block_size, vpb)
2958
+ return self.insert_index_entry(dir_no, mref, fn_bytes)
2959
+ leaf['reals'].insert(leaf['idx'], entry)
2960
+ # fast path: fits in the leaf (grow the root if that is the leaf)
2961
+ if self._pack_or_grow(dir_no, leaf['ref'], leaf['buf'], leaf['hdr'],
2962
+ leaf['rao'], leaf['reals'], leaf['end'], block_size):
2963
+ return True
2964
+
2965
+ # leaf overflow → multi-level atomic cascade split. Plan the whole
2966
+ # cascade bottom-up, allocating one block per split; if any allocation
2967
+ # fails (e.g. $INDEX_ALLOCATION mapping-pairs overflow) free them all and
2968
+ # refuse — nothing else is written, so the refusal is atomic. Only once
2969
+ # every block is reserved do we commit the node writes (which can't fail).
2970
+ if leaf['ref'][0] != 'block':
2971
+ raise NtfsError(errno.EIO, 'root leaf overflow after conversion check')
2972
+ allocated, writes, carry = [], [], None
2973
+ i = len(path) - 1
2974
+ try:
2975
+ while i >= 0:
2976
+ node = path[i]
2977
+ if carry is not None:
2978
+ self._insert_median_inplace(node, *carry)
2979
+ carry = None
2980
+ if node['ref'][0] == 'root': # root: grow or heighten
2981
+ fresh = self._load_record(dir_no)
2982
+ if fresh is None:
2983
+ raise NtfsError(errno.EIO,
2984
+ f'directory record {dir_no} unreadable')
2985
+ ra_f, _l = self._base_attr(fresh, AT_INDEX_ROOT, self._ix.name)
2986
+ body_len = len(b''.join(node['reals']) + node['end'])
2987
+ # keep the resident root small (a third of the record) so the
2988
+ # base has room for $INDEX_ALLOCATION + $BITMAP — heighten to
2989
+ # a tiny END→child pointer early rather than growing a fat 2-level
2990
+ # root that starves the base
2991
+ if body_len <= self._rec_size // 3 \
2992
+ and self._root_body_room(fresh, ra_f, body_len):
2993
+ writes.append(('rootfit', node))
2994
+ else:
2995
+ b_i, b_vcn = self._alloc_index_block(dir_no, block_size, vpb)
2996
+ allocated.append(b_i)
2997
+ writes.append(('rootheighten', node, b_vcn))
2998
+ break
2999
+ if self._would_pack(node['cap'], node['reals'], node['end']):
3000
+ writes.append(('blockfit', node))
3001
+ break
3002
+ end_is_node = bool(struct.unpack_from('<H', node['end'], 12)[0] & 1)
3003
+ mid = len(node['reals']) // 2
3004
+ median_e = node['reals'][mid]
3005
+ left, right = node['reals'][:mid], node['reals'][mid + 1:]
3006
+ if end_is_node:
3007
+ left_end = self._end_node(self._entry_subnode(median_e))
3008
+ right_end = node['end']
3009
+ else:
3010
+ left_end = right_end = self._leaf_end()
3011
+ my_vcn = node['ref'][1] * vpb
3012
+ new_i, new_vcn = self._alloc_index_block(dir_no, block_size, vpb)
3013
+ allocated.append(new_i)
3014
+ writes.append(('split', node, left, left_end, right, right_end, new_vcn))
3015
+ carry = (self._make_node_entry(median_e, my_vcn), my_vcn, new_vcn)
3016
+ i -= 1
3017
+ else:
3018
+ raise NtfsError(errno.EIO, 'cascade ran off the top of the tree')
3019
+ except NtfsError:
3020
+ for bi in allocated:
3021
+ self._free_index_block(dir_no, bi)
3022
+ raise
3023
+
3024
+ for w in writes: # commit — cannot fail
3025
+ if w[0] == 'split':
3026
+ _, node, left, left_end, right, right_end, new_vcn = w
3027
+ if not self._pack_node(node['buf'], node['hdr'], left, left_end):
3028
+ raise NtfsError(errno.EIO, 'left half overflow (unexpected)')
3029
+ self._write_block(dir_no, node['ref'][1], block_size, node['buf'])
3030
+ nb = self._new_indx_block(new_vcn, block_size)
3031
+ if not self._pack_node(nb, 24, right, right_end):
3032
+ raise NtfsError(errno.EIO, 'upper half overflow (unexpected)')
3033
+ self._write_block(dir_no, new_vcn // vpb, block_size, nb)
3034
+ elif w[0] == 'blockfit':
3035
+ node = w[1]
3036
+ self._pack_node(node['buf'], node['hdr'], node['reals'], node['end'])
3037
+ self._write_block(dir_no, node['ref'][1], block_size, node['buf'])
3038
+ elif w[0] == 'rootfit':
3039
+ node = w[1]
3040
+ self._write_root_node(dir_no, node['reals'], node['end'], small=False)
3041
+ elif w[0] == 'rootheighten':
3042
+ _, node, b_vcn = w
3043
+ bblk = self._new_indx_block(b_vcn, block_size)
3044
+ self._pack_node(bblk, 24, node['reals'], node['end'])
3045
+ self._write_block(dir_no, b_vcn // vpb, block_size, bblk)
3046
+ self._write_root_node(dir_no, [], self._end_node(b_vcn), small=False)
3047
+ return True
3048
+
3049
+ def _insert_median_inplace(self, node, median, left_vcn, right_vcn) -> None:
3050
+ '''Absorb a child's promoted median: find the pointer to the child that
3051
+ split (subnode == left_vcn), retarget it to the new right block, and
3052
+ insert the median (→left, the half that stayed) just before it.'''
3053
+ reals, end = node['reals'], node['end']
3054
+ for k, e in enumerate(reals):
3055
+ if struct.unpack_from('<H', e, 12)[0] & 1 \
3056
+ and self._entry_subnode(e) == left_vcn:
3057
+ reals[k] = self._set_subnode(e, right_vcn)
3058
+ reals.insert(k, median)
3059
+ return
3060
+ if struct.unpack_from('<H', end, 12)[0] & 1 \
3061
+ and self._entry_subnode(end) == left_vcn:
3062
+ node['end'] = self._set_subnode(end, right_vcn)
3063
+ reals.append(median)
3064
+ return
3065
+ raise NtfsError(errno.EIO, 'cascade: parent pointer to child not found')
3066
+
3067
+ def _free_index_block(self, dir_no, block_index) -> None:
3068
+ '''Clear an INDX block's $BITMAP bit (the block stays allocated in
3069
+ $INDEX_ALLOCATION — a benign leak a bitmap audit reclaims).'''
3070
+ bm = self._read_i30_bitmap(dir_no)
3071
+ bm[block_index >> 3] &= ~(1 << (block_index & 7))
3072
+ self._write_i30_bitmap(dir_no, bm)
3073
+
3074
+ # -- phase 4c: bulk-load rebuild (torn-index recovery from the MFT census) --
3075
+
3076
+ def _leaf_capacity(self, block_size: int) -> int:
3077
+ entries_off = ((40 + 2 * (1 + block_size // 512) + 7) & ~7) - 24
3078
+ return (block_size - 24) - entries_off # bytes for entries incl END
3079
+
3080
+ def _root_body_room(self, rec, root_attr_off: int, body_len: int) -> bool:
3081
+ vofs = struct.unpack_from('<H', rec, root_attr_off + 0x14)[0]
3082
+ cur_alen = struct.unpack_from('<I', rec, root_attr_off + 4)[0]
3083
+ in_use = struct.unpack_from('<I', rec, 0x18)[0]
3084
+ need_vlen = 16 + 16 + body_len # prefix + INDEX_HEADER + body
3085
+ need_alen = (vofs + need_vlen + 7) & ~7
3086
+ return need_alen - cur_alen <= len(rec) - in_use
3087
+
3088
+ def _write_root_node(self, dir_no, body, end, small: bool) -> None:
3089
+ rec = self._load_record(dir_no)
3090
+ if rec is None:
3091
+ raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
3092
+ ra, _l = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
3093
+ vofs = struct.unpack_from('<H', rec, ra + 0x14)[0]
3094
+ cur_vlen = struct.unpack_from('<I', rec, ra + 0x10)[0]
3095
+ body_bytes = b''.join(body) + end
3096
+ need_vlen = 16 + 16 + len(body_bytes)
3097
+ if need_vlen != cur_vlen:
3098
+ # keep the attribute exactly-sized either way: Windows writes
3099
+ # tight roots, and chkdsk flags slack as an index error
3100
+ self._resident_grow_root(rec, ra, ((need_vlen + 7) & ~7) - cur_vlen)
3101
+ vofs = struct.unpack_from('<H', rec, ra + 0x14)[0]
3102
+ cur_vlen = struct.unpack_from('<I', rec, ra + 0x10)[0]
3103
+ val = ra + vofs # INDEX_ROOT prefix (16) kept
3104
+ struct.pack_into('<III', rec, val + 16, 16, 16 + len(body_bytes), cur_vlen - 16)
3105
+ rec[val + 16 + 12] = 0 if small else 1 # INDEX_HEADER.flags
3106
+ rec[val + 16 + 13:val + 16 + 16] = b'\x00\x00\x00'
3107
+ rec[val + 32:val + 32 + len(body_bytes)] = body_bytes
3108
+ for i in range(val + 32 + len(body_bytes), val + cur_vlen):
3109
+ rec[i] = 0
3110
+ self.write_record(dir_no, rec)
3111
+
3112
+ def _ensure_index_blocks(self, dir_no, block_size, vpb, nblocks) -> None:
3113
+ alloc = self._attr_value(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
3114
+ have = len(alloc) // block_size
3115
+ while have < nblocks:
3116
+ self._grow_index_alloc(dir_no, block_size, vpb,
3117
+ self._read_i30_bitmap(dir_no), have)
3118
+ have += 1
3119
+
3120
+ def _set_i30_bitmap_used(self, dir_no, nblocks) -> None:
3121
+ bm = self._read_i30_bitmap(dir_no)
3122
+ for i in range(len(bm) * 8):
3123
+ if i < nblocks:
3124
+ bm[i >> 3] |= 1 << (i & 7)
3125
+ else:
3126
+ bm[i >> 3] &= ~(1 << (i & 7))
3127
+ self._write_i30_bitmap(dir_no, bm)
3128
+
3129
+ def rebuild_index(self, dir_no: int) -> dict:
3130
+ '''Rebuild a directory's $I30 from the MFT census (chkdsk stage-2 rebuild,
3131
+ native): build sorted $FILE_NAME leaf entries and bulk-load them.'''
3132
+ census = self.children_from_mft(dir_no)['children']
3133
+ items = []
3134
+ for c in census:
3135
+ e = self._ix.build_leaf(c['record'] | (c['seq'] << 48), c['fn_bytes'])
3136
+ items.append((self._ix.entry_sort_key(e), e))
3137
+ items.sort(key=lambda t: t[0])
3138
+ return self._bulk_write_index(dir_no, [e for _, e in items])
3139
+
3140
+ def _bulk_write_index(self, owner: int, entries: list) -> dict:
3141
+ '''Write `entries` (already in self._ix collation order) as the index
3142
+ named self._ix.name on record `owner`: a resident root if they fit,
3143
+ else a bottom-up bulk-loaded tree — one INDX block, or several with
3144
+ the root a single END→internal pointer and separators one level down.
3145
+ Shared by the $I30 directory rebuild and the $SII/$SDH $Secure rebuild;
3146
+ schema-generic — the separator format follows self._ix.'''
3147
+ rec = self._load_record(owner)
3148
+ if rec is None:
3149
+ raise NtfsError(errno.EIO, f'record {owner} unreadable')
3150
+ loc = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
3151
+ if not loc:
3152
+ raise NtfsError(errno.ENOENT, f'record {owner} has no index root')
3153
+ ra, _l = loc
3154
+ rvofs = ra + struct.unpack_from('<H', rec, ra + 0x14)[0]
3155
+ block_size = struct.unpack_from('<I', rec, rvofs + 8)[0]
3156
+ vpb = max(block_size // self._cluster_size, 1)
3157
+ has_ia = self._base_attr(rec, AT_INDEX_ALLOCATION, self._ix.name) is not None
3158
+ leaf_end = self._leaf_end()
3159
+
3160
+ # small enough for the resident root, and no allocation to reconcile?
3161
+ if not has_ia:
3162
+ if self._root_body_room(rec, ra, len(b''.join(entries) + leaf_end)):
3163
+ self._write_root_node(owner, entries, leaf_end, small=True)
3164
+ return {'entries': len(entries), 'blocks': 0}
3165
+ # needs a large index but has none — create $INDEX_ALLOCATION +
3166
+ # $BITMAP, then fall through (the large build overwrites block 0)
3167
+ self._small_to_large(owner, block_size, vpb)
3168
+ if getattr(self, '_ia_cache', None):
3169
+ self._ia_cache.pop(owner, None)
3170
+
3171
+ # large: pack leaves. Root stays a single END→child pointer; separators
3172
+ # live in an internal block one level down (keeps the resident root tiny).
3173
+ cap = self._leaf_capacity(block_size)
3174
+ leaves, cur, cur_len = [], [], 0
3175
+ for e in entries:
3176
+ if cur and cur_len + len(e) + 16 > cap: # +16 keeps END room
3177
+ leaves.append(cur)
3178
+ cur, cur_len = [], 0
3179
+ cur.append(e)
3180
+ cur_len += len(e)
3181
+ leaves.append(cur)
3182
+ nleaves = len(leaves)
3183
+
3184
+ if nleaves == 1: # one leaf block; root → it
3185
+ self._ensure_index_blocks(owner, block_size, vpb, 1)
3186
+ blk = self._new_indx_block(0, block_size)
3187
+ self._pack_node(blk, 24, leaves[0], leaf_end)
3188
+ self._write_block(owner, 0, block_size, blk)
3189
+ self._set_i30_bitmap_used(owner, 1)
3190
+ self._write_root_node(owner, [], self._end_node(0), small=False)
3191
+ return {'entries': len(entries), 'blocks': 1}
3192
+
3193
+ # build the tree bottom-up to arbitrary depth (root stays a single
3194
+ # END→child pointer; leaves are blocks 0..nleaves-1, internal levels
3195
+ # take the blocks above them).
3196
+ leaf_nodes = [{'vcn': i * vpb, 'entries': list(leaves[i]), 'end_vcn': None}
3197
+ for i in range(nleaves)]
3198
+ planned = [] # (block_index, body, end)
3199
+ top_vcn, total = self._build_tree(leaf_nodes, block_size, vpb,
3200
+ nleaves, planned)
3201
+ self._ensure_index_blocks(owner, block_size, vpb, total)
3202
+ for bi, body, end in planned:
3203
+ blk = self._new_indx_block(bi * vpb, block_size)
3204
+ if not self._pack_node(blk, 24, body, end):
3205
+ raise NtfsError(errno.EIO, f'node {bi} body overflow (unexpected)')
3206
+ self._write_block(owner, bi, block_size, blk)
3207
+ self._set_i30_bitmap_used(owner, total)
3208
+ self._write_root_node(owner, [], self._end_node(top_vcn), small=False)
3209
+ return {'entries': len(entries), 'blocks': total}
3210
+
3211
+ def _build_tree(self, nodes, block_size, vpb, next_bi, planned):
3212
+ '''Bottom-up bulk load. `nodes` is an ordered list of
3213
+ {vcn, entries, end_vcn} (end_vcn None marks a leaf). Records each
3214
+ node into `planned` and returns (top_vcn, total_block_count). The
3215
+ parent level is built by promoting the first entry of every non-first
3216
+ sibling as a separator (the standard NTFS B+ convention), recursing
3217
+ until one node remains.'''
3218
+ if len(nodes) == 1:
3219
+ n = nodes[0]
3220
+ end = self._leaf_end() if n['end_vcn'] is None else self._end_node(n['end_vcn'])
3221
+ planned.append((n['vcn'] // vpb, n['entries'], end))
3222
+ return n['vcn'], next_bi
3223
+
3224
+ seps = [] # (first_entry, left_child_vcn)
3225
+ for i in range(1, len(nodes)):
3226
+ seps.append((nodes[i]['entries'].pop(0), nodes[i - 1]['vcn']))
3227
+ final_child = nodes[-1]['vcn']
3228
+ for n in nodes: # this level is now final
3229
+ end = self._leaf_end() if n['end_vcn'] is None else self._end_node(n['end_vcn'])
3230
+ planned.append((n['vcn'] // vpb, n['entries'], end))
3231
+
3232
+ cap = self._leaf_capacity(block_size) - 24 # reserve the node END entry
3233
+ groups, cur, cur_len = [], [], 0
3234
+ for e, child in seps:
3235
+ ne = self._make_node_entry(e, child)
3236
+ if cur and cur_len + len(ne) > cap:
3237
+ groups.append(cur)
3238
+ cur, cur_len = [], 0
3239
+ cur.append((ne, child))
3240
+ cur_len += len(ne)
3241
+ groups.append(cur)
3242
+
3243
+ parents, bi = [], next_bi
3244
+ for j, group in enumerate(groups):
3245
+ end_vcn = final_child if j == len(groups) - 1 else groups[j + 1][0][1]
3246
+ parents.append({'vcn': bi * vpb, 'end_vcn': end_vcn,
3247
+ 'entries': [ne for ne, _c in group]})
3248
+ bi += 1
3249
+ return self._build_tree(parents, block_size, vpb, bi, planned)
3250
+
3251
+ def rebuild_dir(self, path: str) -> dict:
3252
+ '''Adapter so full_fix's rebuild step works on the native engine —
3253
+ returns the {added, duplicates, planned} shape full_fix expects.'''
3254
+ r = self.rebuild_index(self._resolve(path))
3255
+ return {'added': r['entries'], 'duplicates': 0, 'planned': r['entries']}
3256
+
3257
+ def reconnect_lost(self, lost: dict) -> int:
3258
+ '''Lost-file reconnection: re-add each $FILE_NAME leaf entry into
3259
+ the live parent's index.'''
3260
+ self._invalidate_mft_cache()
3261
+ mref = lost['record'] | (lost['seq'] << 48)
3262
+ added = 0
3263
+ for fn in lost['fn_list']:
3264
+ try:
3265
+ self.insert_index_entry(lost['parent'], mref, fn)
3266
+ added += 1
3267
+ except NtfsError as exc:
3268
+ if exc.errno != errno.EEXIST:
3269
+ raise
3270
+ return added
3271
+
3272
+ # -- $LogFile reset (the ntfsfix/chkdsk approach — a reset, NOT a replay) --
3273
+
3274
+ def reset_logfile(self) -> int:
3275
+ '''Empty $LogFile by filling it with 0xff, so the driver and Windows
3276
+ treat the journal as clean and replay nothing. This DISCARDS any
3277
+ pending crash-journal transactions — it is not a replay of them;
3278
+ the structural checks reconcile the on-disk state instead, exactly
3279
+ as chkdsk does (which also resets, not replays, the log).'''
3280
+ size, runs = self._stream_runs(FILE_LOGFILE, AT_DATA, None)
3281
+ if not runs:
3282
+ raise NtfsError(errno.ENOENT, '$LogFile has no non-resident $DATA')
3283
+ chunk = b'\xff' * (1 << 20)
3284
+ off = 0
3285
+ while off < size:
3286
+ self._runs_write(runs, off, chunk[:min(len(chunk), size - off)])
3287
+ off += len(chunk)
3288
+ return size
3289
+
3290
+ def clear_dirty(self) -> None:
3291
+ '''Clear the volume dirty flag — only valid after checks pass clean.'''
3292
+ self._was_dirty = False # so close() does not re-clear or preserve
3293
+ self._set_dirty(False)
3294
+
3295
+
3296
+ # -- phase 5: purge composition (unlink + free record + free clusters) --
3297
+
3298
+ def _extension_records(self, rec_no: int) -> set:
3299
+ '''Extension record numbers a base record's $ATTRIBUTE_LIST references
3300
+ (empty if the file lives in a single record) '''
3301
+
3302
+ rec = self._load_record(rec_no)
3303
+ if rec is None:
3304
+ return set()
3305
+ frozen = bytes(rec)
3306
+ exts, nc = set(), self.nr_clusters()
3307
+ for a, t, ln in _attrs(frozen):
3308
+ if t != AT_ATTRIBUTE_LIST:
3309
+ continue
3310
+ if frozen[a + 8] == 0:
3311
+ vo = struct.unpack_from('<H', frozen, a + 20)[0]
3312
+ vl = struct.unpack_from('<I', frozen, a + 16)[0]
3313
+ listing = frozen[a + vo:a + vo + vl]
3314
+ else:
3315
+ _p, _ph, r = _check_mapping_pairs(frozen, a, ln, nc)
3316
+ size = struct.unpack_from('<q', frozen, a + 48)[0]
3317
+ listing = self._runs_read(r, 0, size)
3318
+ pos = 0
3319
+ while pos + 26 <= len(listing):
3320
+ e_len = struct.unpack_from('<H', listing, pos + 4)[0]
3321
+ if e_len < 26:
3322
+ break
3323
+ mref = struct.unpack_from('<Q', listing, pos + 16)[0] & MREF_MASK
3324
+ if mref != rec_no:
3325
+ exts.add(mref)
3326
+ pos += e_len
3327
+ return exts
3328
+
3329
+
3330
+ def _record_alloc_runs(self, rec_no: int):
3331
+ '''(lcn, run_len) clusters owned by every non-resident attribute of the
3332
+ file — spanning extension records when an $ATTRIBUTE_LIST is present
3333
+ (_record_attrs follows the list) '''
3334
+
3335
+ runs, nc = [], self.nr_clusters()
3336
+ for frozen, a, length in self._record_attrs(rec_no):
3337
+ if frozen[a + 8] == 1: # non-resident
3338
+ _p, _ph, r = _check_mapping_pairs(frozen, a, length, nc)
3339
+ runs += [(lcn, rlen) for lcn, rlen, _v in r]
3340
+ return runs
3341
+
3342
+
3343
+ def _free_file_record(self, rec_no: int) -> None:
3344
+ '''Clear MFT_RECORD_IN_USE, bump the sequence (so stale refs detect as
3345
+ stale), and clear the $MFT bitmap bit '''
3346
+
3347
+ rec = self._load_record(rec_no)
3348
+ if rec is None:
3349
+ raise NtfsError(errno.EIO, f'record {rec_no} unreadable')
3350
+ flags = struct.unpack_from('<H', rec, 22)[0]
3351
+ struct.pack_into('<H', rec, 22, flags & ~1)
3352
+ seq = struct.unpack_from('<H', rec, 16)[0]
3353
+ struct.pack_into('<H', rec, 16, (seq + 1) & 0xFFFF or 1)
3354
+ self.write_record(rec_no, rec)
3355
+ self.free_record(rec_no)
3356
+
3357
+
3358
+ def purge_orphan(self, path: str, name: str, really: bool) -> dict:
3359
+ ''' Reclaim a true orphan natively. crash-safe order : unlink, clear the
3360
+ record's in-use flag, free clusters, so no live record ever references
3361
+ freed space '''
3362
+
3363
+ verdict = self.classify_dirent(path, name)
3364
+ if verdict['state'] != 'orphan':
3365
+ raise SystemExit(f'refusing purge: entry is {verdict["state"]!r} '
3366
+ f'— {verdict["action"]}')
3367
+ if not really:
3368
+ return verdict
3369
+ dir_no = self._resolve(path)
3370
+ rec_no = verdict['dirent_record']
3371
+ runs = self._record_alloc_runs(rec_no) # all clusters (base + extensions)
3372
+ exts = self._extension_records(rec_no) # extension records to free too
3373
+ # crash-safe order: unlink, free every record (in-use cleared) so no
3374
+ # live record references the clusters, then free the clusters
3375
+ self.remove_index_entry(dir_no, name)
3376
+ self._free_file_record(rec_no)
3377
+ for ext in sorted(exts):
3378
+ self._free_file_record(ext)
3379
+ if runs:
3380
+ self.free_clusters(runs)
3381
+ verdict['purged'] = True
3382
+ return verdict
3383
+
3384
+ # -- op 1: the torn-truncate terminator (native) --
3385
+
3386
+ def terminate_mapping_pairs(self, rec_no: int, mp_off: int) -> None:
3387
+ '''Zero the first mapping-pairs byte of a phantom-runs empty attribute.
3388
+ Whole-record sealed write, so fixup slots are no constraint — the
3389
+ sealer regenerates them.'''
3390
+ rec = self._load_record(rec_no)
3391
+ if rec is None:
3392
+ raise NtfsError(errno.EIO, f'record {rec_no} unreadable')
3393
+ if rec[mp_off] == 0:
3394
+ return # already terminated
3395
+ rec[mp_off] = 0
3396
+ self.write_record(rec_no, rec)
3397
+
3398
+
3399
+ def open_volume(device: str, readonly: bool = True):
3400
+ '''Open the volume: reads use RawVolume, writes use RawVolumeRW.'''
3401
+ return RawVolume(device) if readonly else RawVolumeRW(device)
3402
+
3403
+
3404
+ # ── full-volume pass: the chkdsk /f flow (the parts this tool can do) ────────
3405
+
3406
+ def run_surface(device: str, vol: RawVolume, direct: bool = False) -> dict:
3407
+ '''chkdsk /r stage 4, report-only: read every cluster of the volume
3408
+ (sequential 64 MiB chunks, bisecting failures down to clusters), then
3409
+ attribute unreadable clusters to their owning files.
3410
+
3411
+ direct=True opens with O_DIRECT (page-aligned mmap buffer + preadv) so
3412
+ a 500 GB scan does not evict the machine's page cache; falls back to
3413
+ buffered reads + posix_fadvise(DONTNEED) where O_DIRECT is refused.'''
3414
+
3415
+ end = vol.nr_clusters() * vol.cluster_size()
3416
+ fd = None
3417
+ if direct and hasattr(os, 'O_DIRECT'):
3418
+ try:
3419
+ fd = os.open(device, os.O_RDONLY | os.O_DIRECT | _O_BINARY)
3420
+ except OSError:
3421
+ print(' (O_DIRECT refused here — using buffered reads + fadvise)')
3422
+ if fd is None:
3423
+ direct = False
3424
+ fd = os.open(device, os.O_RDONLY | _O_BINARY)
3425
+ last = [0]
3426
+
3427
+ def progress(pos: int, total: int) -> None:
3428
+ if pos - last[0] >= (32 << 30):
3429
+ print(f' ... {pos >> 30} / {total >> 30} GiB read')
3430
+ last[0] = pos
3431
+
3432
+ if direct:
3433
+ import mmap
3434
+ dbuf = mmap.mmap(-1, 64 << 20) # page-aligned, as O_DIRECT requires
3435
+
3436
+ def pread(pos: int, count: int) -> bytes:
3437
+ got = os.preadv(fd, [memoryview(dbuf)[:count]], pos)
3438
+ return dbuf[:got]
3439
+ else:
3440
+ def pread(pos: int, count: int) -> bytes:
3441
+ data = _pread(fd, count, pos)
3442
+ if hasattr(os, 'posix_fadvise'): # don't evict the page cache
3443
+ os.posix_fadvise(fd, pos, count, os.POSIX_FADV_DONTNEED)
3444
+ return data
3445
+
3446
+ try:
3447
+ bad = _surface_scan(pread, end, vol.cluster_size(), progress)
3448
+ finally:
3449
+ os.close(fd)
3450
+ return {'bad': bad, 'bytes': end,
3451
+ 'owners': vol.surface_owners(set(bad)) if bad else []}
3452
+
3453
+
3454
+ def _print_surface(scan: dict, indent: str = '') -> None:
3455
+ print(f'{indent}{scan["bytes"] >> 20} MiB read, '
3456
+ f'{len(scan["bad"])} unreadable cluster(s)')
3457
+ for own in scan['owners'][:10]:
3458
+ heads = ', '.join(str(c) for c in own['clusters'][:6])
3459
+ more = '...' if len(own['clusters']) > 6 else ''
3460
+ if own['record'] is None:
3461
+ print(f'{indent}! {len(own["clusters"])} bad cluster(s) in free space '
3462
+ f'({heads}{more})')
3463
+ else:
3464
+ print(f'{indent}! record {own["record"]} attr {own["attr"]:#x} '
3465
+ f'({own["names"]}): bad cluster(s) {heads}{more}')
3466
+ if scan['bad']:
3467
+ print(f'{indent}report-only: reallocate via a real chkdsk /r, or restore '
3468
+ 'the affected files from backup')
3469
+
3470
+
3471
+ def full_fix(device: str, really: bool, surface: bool = False,
3472
+ direct: bool = False) -> int:
3473
+ '''The chkdsk /f flow, the parts this tool implements. Stage 1 verifies
3474
+ every MFT record and attribute runlist (auto-repairing torn empty-
3475
+ attribute truncates); stage 2 walks every directory index, repairing
3476
+ dangling entries and rebuilding torn indexes; stage 3 validates and
3477
+ repairs $Secure; stage 5 reconciles $Bitmap accounting; the USN journal
3478
+ is validated and reset. Stage 4 (surface scan) is report-only.'''
3479
+
3480
+ extra_issues = extra_fixed = extra_failed = 0
3481
+ with open_volume(device, readonly=not really) as vol:
3482
+ print('stage 1: examining MFT records and attribute runlists ...')
3483
+ survey = vol.mft_survey(want_used=True, want_security=True)
3484
+ health = survey
3485
+ print(f' {health["total"]} records, {health["in_use"]} in use '
3486
+ f'({health["dirs"]} directories), {health["torn"]} torn, '
3487
+ f'{len(health["runlist_problems"])} runlist problem(s)')
3488
+ for prob in health['runlist_problems']:
3489
+ extra_issues += 1
3490
+ print(f'! record {prob["record"]} attr {prob["attr_type"]:#x} '
3491
+ f'({prob["names"]}): {prob["problem"]}')
3492
+ if prob['fix_mp_off'] is None:
3493
+ print(' no safe auto-repair — chkdsk territory')
3494
+ if really:
3495
+ extra_failed += 1
3496
+ continue
3497
+ if not really:
3498
+ print(' (safe auto-repair available: terminate the phantom runlist)')
3499
+ continue
3500
+ try:
3501
+ vol.terminate_mapping_pairs(prob['record'], prob['fix_mp_off'])
3502
+ extra_fixed += 1
3503
+ print(' fixed (runlist terminated; any leaked clusters are '
3504
+ 'reclaimed in stage 5)')
3505
+ except NtfsError as exc:
3506
+ extra_failed += 1
3507
+ print(f' FIX FAILED: {exc}')
3508
+
3509
+ # $MFT's own $BITMAP (bit i = record i allocated): a crash between the
3510
+ # bitmap write and the record write leaks set bits — chkdsk's "the
3511
+ # master file table's (MFT) BITMAP attribute is incorrect". Reconcile
3512
+ # against the records' surveyed in-use flags, the on-disk truth.
3513
+ print('mft record bitmap: verifying ...')
3514
+ mft_bmp = bytes(vol._read_whole_attr(0, AT_BITMAP, None, None) or b'')
3515
+ rec_used = survey['rec_used']
3516
+ nbits = min(survey['n_slots'], len(mft_bmp) * 8)
3517
+ mft_mism = [i for i in range(nbits)
3518
+ if bool(rec_used[i >> 3] & (1 << (i & 7)))
3519
+ != bool(mft_bmp[i >> 3] & (1 << (i & 7)))]
3520
+ if len(mft_bmp) * 8 < survey['n_slots']:
3521
+ extra_issues += 1
3522
+ print(f'! $BITMAP covers {len(mft_bmp) * 8} of '
3523
+ f'{survey["n_slots"]} record slots — truncated (report-only)')
3524
+ if really:
3525
+ extra_failed += 1
3526
+ if mft_mism:
3527
+ extra_issues += 1
3528
+ print(f'! {len(mft_mism)} record(s) disagree with the bitmap '
3529
+ f'(first: {mft_mism[:8]})')
3530
+ if not really:
3531
+ print(' (repairable: --really rewrites the bitmap from the '
3532
+ 'surveyed in-use flags)')
3533
+ else:
3534
+ n = vol.apply_mft_bitmap(rec_used, nbits)
3535
+ again = bytes(vol._read_whole_attr(0, AT_BITMAP, None, None)
3536
+ or b'')
3537
+ left = [i for i in range(nbits)
3538
+ if bool(rec_used[i >> 3] & (1 << (i & 7)))
3539
+ != bool(again[i >> 3] & (1 << (i & 7)))]
3540
+ if left:
3541
+ extra_failed += 1
3542
+ print(' REWRITE VERIFY FAILED')
3543
+ else:
3544
+ extra_fixed += 1
3545
+ print(f' bitmap rewritten ({n} bit(s) corrected)')
3546
+ else:
3547
+ print(' consistent')
3548
+
3549
+ print('stage 2: examining directory indexes ...')
3550
+ walked = dangling = torn = fixed = failed = 0
3551
+ stack, seen = [('/', FILE_ROOT)], {FILE_ROOT}
3552
+ referenced: set[int] = set()
3553
+ while stack:
3554
+ path, dir_no = stack.pop()
3555
+ walked += 1
3556
+ if walked % 2000 == 0:
3557
+ print(f' ... {walked} directories walked, {len(stack)} queued')
3558
+
3559
+ probe = vol.probe_dir_no(dir_no)
3560
+ if not probe['readable']:
3561
+ torn += 1
3562
+ print(f'! torn index: {path!r} — {probe["error"]}')
3563
+ if not really:
3564
+ print(' (children not walked; --really will rebuild this index)')
3565
+ continue
3566
+ try:
3567
+ result = vol.rebuild_dir(path)
3568
+ print(f' rebuilt: {result["added"]} entries re-added')
3569
+ except (SystemExit, NtfsError) as exc:
3570
+ failed += 1
3571
+ print(f' REBUILD FAILED: {exc}')
3572
+ continue
3573
+ probe = vol.probe_dir_no(dir_no)
3574
+ if not probe['readable']:
3575
+ failed += 1
3576
+ print(f' VERIFY FAILED: still unreadable — {probe["error"]}')
3577
+ continue
3578
+ fixed += 1
3579
+
3580
+ for entry in probe['entries']:
3581
+ full = (path.rstrip('/')) + '/' + entry['name']
3582
+ if entry['status'] == 'ok':
3583
+ referenced.add(entry['record'])
3584
+ if entry.get('is_dir') and entry['record'] not in seen:
3585
+ seen.add(entry['record'])
3586
+ stack.append((full, entry['record']))
3587
+ continue
3588
+ dangling += 1
3589
+ try:
3590
+ verdict = vol.classify_dirent(path, entry['name'])
3591
+ except NtfsError as exc:
3592
+ if really and exc.errno == errno.ENOENT:
3593
+ # A purge of a sibling name (WIN32/DOS twin) of the same
3594
+ # file already removed this entry along with it.
3595
+ print(f'! {full!r}: entry already removed with its twin name')
3596
+ fixed += 1
3597
+ else:
3598
+ failed += 1
3599
+ print(f'! {full!r}: triage failed: {exc}')
3600
+ continue
3601
+ print(f'! {full!r}: {verdict["state"]} -> {verdict["action"]}')
3602
+ if not really or verdict['state'] == 'healthy':
3603
+ continue
3604
+ try:
3605
+ if verdict['state'] == 'orphan':
3606
+ vol.purge_orphan(path, entry['name'], really=True)
3607
+ else:
3608
+ vol.remove_dirent(path, entry['name'], really=True)
3609
+ fixed += 1
3610
+ print(' fixed')
3611
+ except (SystemExit, NtfsError) as exc:
3612
+ failed += 1
3613
+ print(f' FIX FAILED: {exc}')
3614
+
3615
+ print('lost files: records no index entry references ...')
3616
+ lost = vol.find_lost_files(referenced)
3617
+ print(f' {len(lost)} lost file(s)')
3618
+ for lf in lost:
3619
+ extra_issues += 1
3620
+ where = (f"live parent {lf['parent']}" if lf['parent_ok']
3621
+ else f"parent {lf['parent']} is dead/reused — not reconnectable")
3622
+ print(f"! lost file {lf['name']!r} (record {lf['record']}): {where}")
3623
+ if not lf['parent_ok']:
3624
+ if really:
3625
+ extra_failed += 1
3626
+ print(' chkdsk territory (found.000-style recovery not implemented)')
3627
+ continue
3628
+ if not really:
3629
+ print(' (repairable: --really re-adds it to its parent directory)')
3630
+ continue
3631
+ try:
3632
+ added = vol.reconnect_lost(lf)
3633
+ if added:
3634
+ extra_fixed += 1
3635
+ print(f' reconnected ({added} index entr'
3636
+ f'{"y" if added == 1 else "ies"} re-added)')
3637
+ else:
3638
+ extra_failed += 1
3639
+ print(' NOT reconnected: name already taken in the parent')
3640
+ except NtfsError as exc:
3641
+ extra_failed += 1
3642
+ print(f' RECONNECT FAILED: {exc}')
3643
+
3644
+ print('stage 3: verifying security descriptors ($Secure) ...')
3645
+ sec = vol.secure_check(survey=survey)
3646
+ for note in sec['notes']:
3647
+ print(f' note: {note}')
3648
+ if sec['present']:
3649
+ print(f' {sec["descriptors"]} descriptors, {sec["files_checked"]} files '
3650
+ f'checked: {len(sec["mirror_fixes"])} mirror mismatch(es), '
3651
+ f'{len(sec["index_problems"])} index problem(s), '
3652
+ f'{len(sec["problems"])} unrepairable, '
3653
+ f'{len(sec["ref_missing"])} dangling reference(s)')
3654
+ for prob in (sec['problems'] + sec['index_problems']
3655
+ + sec['ref_missing'])[:10]:
3656
+ print(f'! {prob}')
3657
+ sec_issues = (bool(sec['mirror_fixes']) + bool(sec['index_problems'])
3658
+ + bool(sec['problems']) + bool(sec['ref_missing']))
3659
+ extra_issues += sec_issues
3660
+ if sec_issues and really:
3661
+ if sec['mirror_fixes']:
3662
+ try:
3663
+ n = vol.secure_fix_mirrors(sec['mirror_fixes'])
3664
+ extra_fixed += 1
3665
+ print(f' {n} $SDS mirror pair(s) repaired from the good copy')
3666
+ except NtfsError as exc:
3667
+ extra_failed += 1
3668
+ print(f' MIRROR FIX FAILED: {exc}')
3669
+ if sec['index_problems']:
3670
+ try:
3671
+ result = vol.rebuild_secure_indexes(sec['sds_entries'])
3672
+ again = vol.secure_check()
3673
+ if again['index_problems']:
3674
+ extra_failed += 1
3675
+ print(' REBUILD VERIFY FAILED: '
3676
+ + again['index_problems'][0])
3677
+ else:
3678
+ extra_fixed += 1
3679
+ print(f' $SII/$SDH rebuilt from $SDS '
3680
+ f'({result["added"]} descriptors)')
3681
+ except NtfsError as exc:
3682
+ extra_failed += 1
3683
+ print(f' REBUILD FAILED: {exc}')
3684
+ if sec['problems'] or sec['ref_missing']:
3685
+ extra_failed += bool(sec['problems']) + bool(sec['ref_missing'])
3686
+ print(' unrepairable descriptor damage / dangling references — '
3687
+ 'chkdsk territory')
3688
+ elif sec_issues:
3689
+ print(' (repairable with --really: mirror fixes + $SII/$SDH rebuild; '
3690
+ 'dangling references are report-only)')
3691
+
3692
+ if surface:
3693
+ print('stage 4: surface scan — reading every cluster ...')
3694
+ scan = run_surface(device, vol, direct)
3695
+ _print_surface(scan, indent=' ')
3696
+ if scan['bad']:
3697
+ extra_issues += 1
3698
+ if really:
3699
+ extra_failed += 1 # report-only: never repaired here
3700
+
3701
+ print('stage 5: verifying $Bitmap cluster accounting ...')
3702
+ # the shared survey's used-map is stale the moment any repair writes
3703
+ # (purge frees clusters, rebuild allocates INDX blocks) — recompute
3704
+ if really and (fixed + extra_fixed):
3705
+ audit = vol.cluster_audit()
3706
+ else:
3707
+ audit = vol.cluster_audit(survey=survey)
3708
+ for failure in audit['failures']:
3709
+ print(f'! {failure}')
3710
+ print(f' {audit["used_count"]} clusters referenced, '
3711
+ f'{audit["bitmap_count"]} marked in $Bitmap: '
3712
+ f'{audit["extra"]} leaked bit(s), {audit["missing"]} missing bit(s)')
3713
+ if audit['missing']:
3714
+ print(' WARNING: missing bits = live data on clusters marked free — '
3715
+ 'writes to this volume are unsafe until repaired')
3716
+ if audit['extra'] or audit['missing']:
3717
+ extra_issues += 1
3718
+ if not really:
3719
+ print(' (repairable: --really rewrites $Bitmap with the computed map)')
3720
+ elif audit['failures']:
3721
+ extra_failed += 1
3722
+ print(' NOT rewriting $Bitmap — the computed map is incomplete '
3723
+ '(see failures above)')
3724
+ else:
3725
+ vol.apply_bitmap(audit)
3726
+ ondisk = vol._read_whole_attr(FILE_BITMAP, AT_DATA)
3727
+ n = len(audit['used_bytes'])
3728
+ mask = (1 << audit['nr_clusters']) - 1
3729
+ if (int.from_bytes(ondisk[:n], 'little') & mask
3730
+ == int.from_bytes(audit['used_bytes'], 'little') & mask):
3731
+ extra_fixed += 1
3732
+ print(' $Bitmap rewritten and verified')
3733
+ else:
3734
+ extra_failed += 1
3735
+ print(' VERIFY FAILED: $Bitmap readback does not match')
3736
+
3737
+ print('usn journal: verifying ...')
3738
+ usn = vol.usn_check()
3739
+ if not usn['present']:
3740
+ print(' none (journal disabled) — nothing to check')
3741
+ else:
3742
+ print(f' id {usn["journal_id"] or 0:#x}, valid range '
3743
+ f'[{usn["lowest"]}, {usn["next_usn"]}), {usn["records"]} records, '
3744
+ f'{len(usn["problems"])} problem(s)')
3745
+ for note in usn['notes']:
3746
+ print(f' note: {note}')
3747
+ for prob in usn['problems']:
3748
+ print(f'! {prob}')
3749
+ if usn['problems']:
3750
+ extra_issues += 1
3751
+ if not really:
3752
+ print(' (repairable: --really resets the journal; '
3753
+ 'indexers/backup tools rescan)')
3754
+ else:
3755
+ try:
3756
+ vol.usn_reset()
3757
+ again = vol.usn_check()
3758
+ if again['problems'] or again['next_usn'] != 0:
3759
+ extra_failed += 1
3760
+ print(' RESET VERIFY FAILED')
3761
+ else:
3762
+ extra_fixed += 1
3763
+ print(' journal reset (new id; consumers will rescan)')
3764
+ except NtfsError as exc:
3765
+ extra_failed += 1
3766
+ print(f' RESET FAILED: {exc}')
3767
+
3768
+ print('view indexes: $Reparse / $ObjId ...')
3769
+ vw = vol.view_index_check(survey)
3770
+ for note in vw['notes']:
3771
+ print(f' note: {note}')
3772
+ print(f' {vw["checked"]} entr{"y" if vw["checked"] == 1 else "ies"} checked, '
3773
+ f'{len(vw["problems"])} problem(s)')
3774
+ for prob in vw['problems'][:10]:
3775
+ print(f'! {prob}')
3776
+ if vw['problems']:
3777
+ extra_issues += 1
3778
+ if really:
3779
+ extra_failed += 1
3780
+ print(' report-only (rebuild-from-records is a future repair)')
3781
+
3782
+ # -- crashed-session residue: the dirty flag + stale $LogFile --
3783
+ # chkdsk /f ends by resetting the log and clearing the dirty bit; a
3784
+ # volume left flagged is re-checked by Windows at boot and refused
3785
+ # outright by Linux ntfs3. Safe only once every finding above was
3786
+ # repaired — anything unresolved keeps the flag (and the recheck).
3787
+ # In --really mode the flag reads set because RawVolumeRW itself sets
3788
+ # it on open, so the pre-existing state is _was_dirty.
3789
+ print('dirty flag / $LogFile: ...')
3790
+ if (vol._was_dirty if really else vol._dirty_flag()):
3791
+ extra_issues += 1
3792
+ print('! volume is marked dirty (crashed session; $LogFile not clean)')
3793
+ unresolved = (dangling + torn + extra_issues - 1) - (fixed + extra_fixed)
3794
+ if not really:
3795
+ print(' (repairable: --really resets $LogFile and clears the '
3796
+ 'flag once every finding above is repaired)')
3797
+ elif unresolved:
3798
+ extra_failed += 1
3799
+ print(f' left dirty: {unresolved} finding(s) unresolved — '
3800
+ 'the next mount should still trigger a full check')
3801
+ else:
3802
+ vol.reset_logfile()
3803
+ vol.clear_dirty()
3804
+ if vol._dirty_flag():
3805
+ extra_failed += 1
3806
+ print(' CLEAR VERIFY FAILED')
3807
+ else:
3808
+ extra_fixed += 1
3809
+ print(' $LogFile reset (0xff) + dirty flag cleared — '
3810
+ 'the volume mounts clean')
3811
+ else:
3812
+ print(' clean')
3813
+
3814
+ issues = dangling + torn + extra_issues
3815
+ fixed += extra_fixed
3816
+ print(f'-- {walked} directories walked: {dangling} dangling '
3817
+ f'entr{"y" if dangling == 1 else "ies"}, {torn} torn '
3818
+ f'ind{"ex" if torn == 1 else "exes"}'
3819
+ + (f', {extra_issues} record/bitmap issue(s)' if extra_issues else '')
3820
+ + (f'; {fixed} fixed, {issues - fixed} remaining' if really else ''))
3821
+ if health['torn']:
3822
+ print(f'note: {health["torn"]} MFT records are torn — their files are '
3823
+ 'beyond index repair (chkdsk / restore from backup)')
3824
+ covered = ('records, runlists, indexes, security descriptors, $Bitmap and '
3825
+ 'the USN journal are checked')
3826
+ print(f'note: {covered}'
3827
+ + ('; the surface scan ran report-only' if surface
3828
+ else '; add --surface (or use /r) for the stage-4 read of every cluster'))
3829
+ if issues and not really:
3830
+ print('DRY RUN: re-run with --really to repair the findings listed above')
3831
+ return 1 if (issues if not really else issues - fixed) else 0
3832
+
3833
+
3834
+ # ── targeted pre-repair backup ───────────────────────────────────────────────
3835
+
3836
+ def do_backup(device: str, outdir: str, dirs: list[str]) -> int:
3837
+ '''Dump the boot sectors, the full raw $MFT, $MFTMirr, and each given
3838
+ directory's raw $I30 $INDEX_ALLOCATION — everything the repair commands
3839
+ can touch, restorable by hand. Metadata only; no file data.'''
3840
+
3841
+ os.makedirs(outdir, exist_ok=True)
3842
+ with open(device, 'rb') as dev, open(f'{outdir}/boot-16k.bin', 'wb') as fh:
3843
+ fh.write(dev.read(16384))
3844
+ print('boot : 16384 bytes')
3845
+
3846
+ with open_volume(device) as vol:
3847
+ rec_size = vol.mft_record_size()
3848
+ with open(f'{outdir}/mft.bin', 'wb') as fh:
3849
+ offset = 0
3850
+ while True:
3851
+ chunk = vol._mft_bulk(offset, 1 << 20)
3852
+ if not chunk:
3853
+ break
3854
+ fh.write(chunk)
3855
+ offset += len(chunk)
3856
+ print(f'$MFT : {offset} bytes ({offset // rec_size} records)')
3857
+
3858
+ mirror = vol._read_whole_attr(1, AT_DATA) # $MFTMirr
3859
+ with open(f'{outdir}/mftmirr.bin', 'wb') as fh:
3860
+ fh.write(mirror)
3861
+ print(f'$MFTMirr : {len(mirror)} bytes')
3862
+
3863
+ for path in dirs:
3864
+ mft_no = vol.resolve(path)
3865
+ total = -1
3866
+ try:
3867
+ blob = vol._read_whole_attr(mft_no, AT_INDEX_ALLOCATION, INDEX_I30, 4)
3868
+ with open(f'{outdir}/dir-{mft_no}-indx.bin', 'wb') as fh:
3869
+ fh.write(blob)
3870
+ total = len(blob)
3871
+ except NtfsError:
3872
+ pass # resident index only
3873
+ with open(f'{outdir}/dir-{mft_no}-path.txt', 'w') as fh:
3874
+ fh.write(path + '\n')
3875
+ state = '(resident index only)' if total < 0 else f'INDX {total} bytes'
3876
+ print(f'dir-{mft_no:<8}: {state} {path}')
3877
+
3878
+ total = 0
3879
+ with os.scandir(outdir) as it:
3880
+ entries = sorted((e for e in it if e.is_file() and e.name != 'SHA256SUMS'),
3881
+ key=lambda e: e.name)
3882
+ with open(f'{outdir}/SHA256SUMS', 'w') as sums:
3883
+ for entry in entries:
3884
+ digest = hashlib.sha256(open(entry.path, 'rb').read()).hexdigest()
3885
+ sums.write(f'{digest} {entry.name}\n')
3886
+ total += entry.stat().st_size # cached by scandir — no extra stat()
3887
+ print(f'backup complete: {total / 1e6:.0f} MB in {outdir}')
3888
+ return 0
3889
+
3890
+
3891
+ # ── partition discovery: `list` (raw table scan, nothing is mounted) ─────────
3892
+
3893
+ def _human(n: int) -> str:
3894
+ for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'):
3895
+ if n < 1024 or unit == 'TiB':
3896
+ return f'{n:.1f} {unit}' if unit != 'B' else f'{n} B'
3897
+ n /= 1024
3898
+
3899
+
3900
+ def _parse_partitions(fd) -> list[dict]:
3901
+ '''Parse the device's partitioning from raw sector reads: a bare NTFS
3902
+ volume (no table), an MBR (incl. the EBR chain of logical partitions),
3903
+ or a GPT behind its protective MBR. Returns [{index, offset, length,
3904
+ kind, name}]; nothing is mounted and nothing is written.'''
3905
+ sec0 = _pread(fd, 512, 0)
3906
+ if len(sec0) < 512:
3907
+ return []
3908
+ if sec0[3:7] == b'NTFS': # bare volume, no table
3909
+ bps = struct.unpack_from('<H', sec0, 11)[0] or 512
3910
+ sectors = struct.unpack_from('<Q', sec0, 40)[0]
3911
+ return [{'index': 0, 'offset': 0, 'length': (sectors + 1) * bps,
3912
+ 'kind': 'bare', 'name': ''}]
3913
+ if sec0[510:512] != b'\x55\xaa':
3914
+ return []
3915
+ # entry: boot flag, CHS start (3), type, CHS end (3), start LBA, sectors
3916
+ entries = [(sec0[0x1BE + 16 * i + 4],
3917
+ *struct.unpack_from('<II', sec0, 0x1BE + 16 * i + 8))
3918
+ for i in range(4)] # (type, start_lba, sectors)
3919
+ if any(t == 0xEE for t, _s, _n in entries): # protective MBR → GPT
3920
+ hdr = _pread(fd, 512, 512)
3921
+ if hdr[:8] != b'EFI PART':
3922
+ return []
3923
+ arr_lba = struct.unpack_from('<Q', hdr, 72)[0]
3924
+ num, esize = struct.unpack_from('<II', hdr, 80)
3925
+ # clamp what a CORRUPT header can make us read: the spec minimum is
3926
+ # 128 entries of 128 bytes; nothing sane exceeds 512 entries
3927
+ if not 0 < esize <= 4096:
3928
+ return []
3929
+ num = min(num, 512)
3930
+ blob = _pread(fd, -(-num * esize // 512) * 512, arr_lba * 512)
3931
+ out = []
3932
+ for i in range(num):
3933
+ e = blob[i * esize:(i + 1) * esize]
3934
+ if len(e) < 128 or e[:16] == b'\x00' * 16:
3935
+ continue
3936
+ start, end = struct.unpack_from('<QQ', e, 32)
3937
+ name = e[56:128].decode('utf-16-le', 'replace').rstrip('\x00')
3938
+ out.append({'index': i + 1, 'offset': start * 512,
3939
+ 'length': (end - start + 1) * 512,
3940
+ 'kind': 'gpt', 'name': name})
3941
+ return out
3942
+ out = []
3943
+ for i, (ptype, start, sectors) in enumerate(entries):
3944
+ if not ptype:
3945
+ continue
3946
+ if ptype in (0x05, 0x0F, 0x85): # extended → walk the EBRs
3947
+ link, idx = 0, 5
3948
+ seen = set() # corrupt chains can cycle
3949
+ while link not in seen and len(seen) < 128:
3950
+ seen.add(link)
3951
+ ebr = _pread(fd, 512, (start + link) * 512)
3952
+ if len(ebr) < 512 or ebr[510:512] != b'\x55\xaa':
3953
+ break
3954
+ t0 = ebr[0x1BE + 4]
3955
+ s0, n0 = struct.unpack_from('<II', ebr, 0x1BE + 8)
3956
+ if t0:
3957
+ out.append({'index': idx,
3958
+ 'offset': (start + link + s0) * 512,
3959
+ 'length': n0 * 512, 'kind': 'mbr',
3960
+ 'name': f'type {t0:#04x}'})
3961
+ idx += 1
3962
+ t1 = ebr[0x1BE + 16 + 4]
3963
+ s1 = struct.unpack_from('<I', ebr, 0x1BE + 16 + 8)[0]
3964
+ if not t1:
3965
+ break
3966
+ link = s1 # next EBR, extended-relative
3967
+ continue
3968
+ out.append({'index': i + 1, 'offset': start * 512,
3969
+ 'length': sectors * 512, 'kind': 'mbr',
3970
+ 'name': f'type {ptype:#04x}'})
3971
+ return out
3972
+
3973
+
3974
+ def _candidate_devices() -> list[str]:
3975
+ '''Whole-disk device nodes to scan when none were given. Read-only probes;
3976
+ missing/unopenable candidates are simply skipped.'''
3977
+ if sys.platform.startswith('linux'):
3978
+ try:
3979
+ return ['/dev/' + n for n in sorted(os.listdir('/sys/block'))
3980
+ if not n.startswith(('ram', 'zram'))]
3981
+ except OSError:
3982
+ return []
3983
+ if sys.platform == 'win32':
3984
+ return ['\\\\.\\PhysicalDrive' + str(i) for i in range(16)]
3985
+ if sys.platform == 'darwin':
3986
+ import re
3987
+ return ['/dev/' + n for n in sorted(os.listdir('/dev'))
3988
+ if re.fullmatch(r'disk\d+', n)]
3989
+ import re # the BSDs
3990
+ return ['/dev/' + n for n in sorted(os.listdir('/dev'))
3991
+ if re.fullmatch(r'(ada|da|vtbd|nvd|nda)\d+', n)]
3992
+
3993
+
3994
+ def list_volumes(devices: list[str] | None) -> int:
3995
+ '''`list`: find NTFS partitions by reading partition tables + boot sectors
3996
+ straight off the devices — nothing needs to be (or gets) mounted. With
3997
+ no arguments, scans the platform's whole-disk devices; arguments may be
3998
+ device nodes or image files.'''
3999
+ explicit = devices is not None
4000
+ found = 0
4001
+ for dev in (devices if explicit else _candidate_devices()):
4002
+ try:
4003
+ fd = os.open(dev, os.O_RDONLY | _O_BINARY)
4004
+ except FileNotFoundError:
4005
+ if explicit:
4006
+ print(f'{dev}: not found')
4007
+ continue
4008
+ except (PermissionError, OSError) as exc:
4009
+ print(f'{dev}: cannot open read-only ({exc.strerror or exc}) — '
4010
+ 'root/admin is needed for raw device reads')
4011
+ continue
4012
+ try:
4013
+ parts = _parse_partitions(fd)
4014
+ finally:
4015
+ os.close(fd)
4016
+ if not parts:
4017
+ if explicit:
4018
+ print(f'{dev}: no NTFS boot sector and no partition table')
4019
+ continue
4020
+ for p in parts:
4021
+ try:
4022
+ boot = None
4023
+ fd = os.open(dev, os.O_RDONLY | _O_BINARY)
4024
+ try:
4025
+ boot = _pread(fd, 512, p['offset'])
4026
+ finally:
4027
+ os.close(fd)
4028
+ except OSError:
4029
+ continue
4030
+ if boot[3:7] != b'NTFS':
4031
+ continue
4032
+ found += 1
4033
+ state = 'unreadable'
4034
+ label = ''
4035
+ try:
4036
+ # the engine's own open-time dirty warning (stderr) is
4037
+ # redundant here — the row already says DIRTY
4038
+ import contextlib, io
4039
+ with contextlib.redirect_stderr(io.StringIO()):
4040
+ with RawVolume(dev, base=p['offset']) as v:
4041
+ label = v.volume_label()
4042
+ dirty = v._dirty_flag()
4043
+ state = ('DIRTY — a crashed session; /f repairs'
4044
+ if dirty else 'clean')
4045
+ except (NtfsError, OSError) as exc:
4046
+ state = f'DAMAGED ({exc}) — a repair candidate'
4047
+ where = ('' if p['kind'] == 'bare'
4048
+ else f' partition {p["index"]} @ {p["offset"]:#x}')
4049
+ name = f' [{p["name"]}]' if p['name'] and p['kind'] == 'gpt' else ''
4050
+ print(f'{dev}{where} ntfs '
4051
+ + (f'{label!r} ' if label else '')
4052
+ + f'{_human(p["length"])}{name} {state}')
4053
+ if not found:
4054
+ print('no NTFS volumes found' + ('' if explicit else
4055
+ ' (no read access to any disk? try with root/admin)'))
4056
+ else:
4057
+ print(f'-- {found} NTFS volume(s); this listing mounted nothing and '
4058
+ 'wrote nothing')
4059
+ return 0
4060
+
4061
+
4062
+ # ── CLI ──────────────────────────────────────────────────────────────────────
4063
+
4064
+ def main() -> int:
4065
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
4066
+ sub = parser.add_subparsers(dest='cmd', required=True)
4067
+
4068
+ p_scan = sub.add_parser('scan', help='read-only: list a directory and health-check every entry')
4069
+ p_scan.add_argument('device')
4070
+ p_scan.add_argument('dir')
4071
+
4072
+ p_rm = sub.add_parser('rm', help='remove ONE dangling index entry (dry-run unless --really)')
4073
+ p_rm.add_argument('device')
4074
+ p_rm.add_argument('dir')
4075
+ p_rm.add_argument('name')
4076
+ p_rm.add_argument('--really', action='store_true', help='actually write; default is dry-run')
4077
+
4078
+ p_inspect = sub.add_parser('inspect', help='read-only triage: free record, reused record, or true orphan?')
4079
+ p_inspect.add_argument('device')
4080
+ p_inspect.add_argument('dir')
4081
+ p_inspect.add_argument('name')
4082
+
4083
+ p_purge = sub.add_parser('purge', help='fully reclaim a TRUE ORPHAN: dirent + record + clusters (dry-run unless --really)')
4084
+ p_purge.add_argument('device')
4085
+ p_purge.add_argument('dir')
4086
+ p_purge.add_argument('name')
4087
+ p_purge.add_argument('--really', action='store_true', help='actually write; default is dry-run')
4088
+
4089
+ p_rebuild = sub.add_parser('rebuild', help='rebuild a TORN directory index from the MFT (dry-run unless --really)')
4090
+ p_rebuild.add_argument('device')
4091
+ p_rebuild.add_argument('dir')
4092
+ p_rebuild.add_argument('--really', action='store_true', help='actually write; default is dry-run')
4093
+ p_rebuild.add_argument('--force', action='store_true', help='rebuild even if the index currently reads fine')
4094
+
4095
+ p_fix = sub.add_parser('fix', aliases=['/f', '/F', '/r', '/R'],
4096
+ help='chkdsk /f flow: verify MFT records + runlists, walk '
4097
+ 'EVERY directory index, validate $Secure, reconcile '
4098
+ '$Bitmap, check the USN journal (dry-run unless '
4099
+ '--really); /r or --surface adds the stage-4 read '
4100
+ 'of every cluster')
4101
+ p_fix.add_argument('device')
4102
+ p_fix.add_argument('--really', action='store_true', help='actually write; default is dry-run')
4103
+ p_fix.add_argument('--surface', action='store_true',
4104
+ help='also read every cluster (chkdsk /r style, report-only)')
4105
+ p_fix.add_argument('--direct', action='store_true',
4106
+ help='surface scan with O_DIRECT (do not pollute the page cache)')
4107
+
4108
+ p_surface = sub.add_parser('surface', help='stage 4 only: read every cluster and '
4109
+ 'attribute unreadable ones to their files '
4110
+ '(always report-only)')
4111
+ p_surface.add_argument('device')
4112
+ p_surface.add_argument('--direct', action='store_true',
4113
+ help='read with O_DIRECT (do not pollute the page cache)')
4114
+
4115
+ p_backup = sub.add_parser('backup', help='dump boot sectors, $MFT, $MFTMirr and given '
4116
+ "directories' raw $I30 indexes to a directory")
4117
+ p_backup.add_argument('device')
4118
+ p_backup.add_argument('outdir')
4119
+ p_backup.add_argument('dirs', nargs='*', help='directories whose $INDEX_ALLOCATION to dump')
4120
+
4121
+ p_usn = sub.add_parser('usn', help='validate the USN change journal; --really resets a '
4122
+ 'corrupt journal (--reset forces the reset)')
4123
+ p_usn.add_argument('device')
4124
+ p_usn.add_argument('--really', action='store_true', help='actually write; default is dry-run')
4125
+ p_usn.add_argument('--reset', action='store_true', help='reset even if the journal is consistent')
4126
+
4127
+ p_secure = sub.add_parser('secure', help='validate $Secure: $SDS descriptors + mirrors, '
4128
+ '$SII/$SDH indexes, per-file references '
4129
+ '(--really repairs mirrors and rebuilds indexes)')
4130
+ p_secure.add_argument('device')
4131
+ p_secure.add_argument('--really', action='store_true', help='actually write; default is dry-run')
4132
+
4133
+ p_list = sub.add_parser('list', help='find NTFS partitions: raw partition-table + '
4134
+ 'boot-sector scan of the local disks (or the '
4135
+ 'given devices/images); mounts nothing')
4136
+ p_list.add_argument('devices', nargs='*',
4137
+ help='devices or image files to scan (default: all local disks)')
4138
+
4139
+ args = parser.parse_args()
4140
+ if args.cmd == 'list': # read-only by construction; may target mounted disks
4141
+ return list_volumes(args.devices or None)
4142
+ if args.cmd in ('/r', '/R'):
4143
+ args.surface = True
4144
+ if args.cmd in ('/f', '/F', '/r', '/R'):
4145
+ args.cmd = 'fix'
4146
+ assert_not_mounted(args.device)
4147
+
4148
+ if args.cmd == 'fix':
4149
+ return full_fix(args.device, args.really, args.surface, args.direct)
4150
+
4151
+ if args.cmd == 'surface':
4152
+ with open_volume(args.device, readonly=True) as volume:
4153
+ scan = run_surface(args.device, volume, args.direct)
4154
+ _print_surface(scan)
4155
+ return 1 if scan['bad'] else 0
4156
+
4157
+ if args.cmd == 'backup':
4158
+ return do_backup(args.device, args.outdir, args.dirs)
4159
+
4160
+ if args.cmd == 'secure':
4161
+ with open_volume(args.device, readonly=not args.really) as volume:
4162
+ chk = volume.secure_check()
4163
+ for note in chk['notes']:
4164
+ print(f'note: {note}')
4165
+ if not chk['present']:
4166
+ return 0
4167
+ print(f'descriptors: {chk["descriptors"]} in $SDS, '
4168
+ f'{chk["files_checked"]} files reference them')
4169
+ findings = (chk['problems'] + [f'mirror mismatch at {p} ({s} side is good)'
4170
+ for p, _l, s in chk['mirror_fixes']]
4171
+ + chk['index_problems'] + chk['ref_missing'])
4172
+ for prob in findings:
4173
+ print(f'! {prob}')
4174
+ if not findings:
4175
+ print('$Secure consistent')
4176
+ return 0
4177
+ if not args.really:
4178
+ print('DRY RUN: re-run with --really to repair mirrors and rebuild '
4179
+ '$SII/$SDH (unrepairable damage and dangling references are '
4180
+ 'report-only)')
4181
+ return 1
4182
+ if chk['mirror_fixes']:
4183
+ n = volume.secure_fix_mirrors(chk['mirror_fixes'])
4184
+ print(f'{n} $SDS mirror pair(s) repaired')
4185
+ if chk['index_problems']:
4186
+ try:
4187
+ result = volume.rebuild_secure_indexes(chk['sds_entries'])
4188
+ print(f'$SII/$SDH rebuilt from $SDS ({result["added"]} descriptors)')
4189
+ except NtfsError as exc:
4190
+ print(f'$SII/$SDH not rebuilt: {exc}')
4191
+ again = volume.secure_check()
4192
+ remaining = (again['problems'] + again['index_problems']
4193
+ + [f'mirror mismatch at {p}' for p, _l, _s in again['mirror_fixes']]
4194
+ + again['ref_missing'])
4195
+ print(f'verify: {len(remaining)} finding(s) remain')
4196
+ for prob in remaining[:10]:
4197
+ print(f'! {prob}')
4198
+ return 1 if remaining else 0
4199
+
4200
+ if args.cmd == 'usn':
4201
+ with open_volume(args.device, readonly=not args.really) as volume:
4202
+ info = volume.usn_check()
4203
+ if not info['present']:
4204
+ print('no USN journal on this volume (nothing to check)')
4205
+ return 0
4206
+ print(f'journal id : {info["journal_id"] or 0:#x}')
4207
+ print(f'valid range: [{info["lowest"]}, {info["next_usn"]}), '
4208
+ f'{info["records"]} records walked')
4209
+ for note in info['notes']:
4210
+ print(f'note: {note}')
4211
+ for prob in info['problems']:
4212
+ print(f'! {prob}')
4213
+ if not info['problems'] and not args.reset:
4214
+ print('journal consistent')
4215
+ return 0
4216
+ if not args.really:
4217
+ print('DRY RUN: re-run with --really to reset the journal '
4218
+ '(new id; indexers/backup tools rescan)')
4219
+ return 1 if info['problems'] else 0
4220
+ new_id = volume.usn_reset()
4221
+ check = volume.usn_check()
4222
+ ok = not check['problems'] and check['next_usn'] == 0
4223
+ print(f'journal reset: new id {new_id:#x}'
4224
+ + ('' if ok else ' — VERIFY FAILED'))
4225
+ return 0 if ok else 1
4226
+
4227
+ if args.cmd == 'rebuild':
4228
+ with open_volume(args.device, readonly=True) as volume:
4229
+ probe = volume.probe_dir(args.dir)
4230
+ census = volume.children_from_mft(volume.resolve(args.dir))
4231
+
4232
+ if probe['readable']:
4233
+ print(f'index : readable, {len(probe["entries"])} entries '
4234
+ f'({sum(e["status"] != "ok" for e in probe["entries"])} dangling)')
4235
+ else:
4236
+ print(f'index : UNREADABLE (torn) — {probe["error"]}')
4237
+ print(f'mft census: {len(census["children"])} names claim this directory as parent '
4238
+ f'({census["stale_parent"]} stale-parent names and '
4239
+ f'{census["unreadable_records"]} torn MFT records skipped)')
4240
+ for child in census['children']:
4241
+ kind = {1: 'win32', 2: 'dos', 3: 'win32+dos'}.get(child['type'], 'posix')
4242
+ print(f' {child["name"]!r:60} mft={child["record"]} seq={child["seq"]} [{kind}]')
4243
+
4244
+ if probe['readable'] and not args.force:
4245
+ print('index reads fine — refusing to rebuild without --force')
4246
+ return 1
4247
+ if not args.really:
4248
+ print('DRY RUN: re-run with --really to wipe the $I30 index and rebuild it from these names')
4249
+ return 0
4250
+
4251
+ with open_volume(args.device, readonly=False) as volume:
4252
+ result = volume.rebuild_dir(args.dir)
4253
+ print(f'rebuilt: {result["added"]} entries re-added '
4254
+ f'({result["duplicates"]} duplicates skipped, {result["planned"]} planned)')
4255
+
4256
+ with open_volume(args.device, readonly=True) as volume:
4257
+ verify = volume.probe_dir(args.dir)
4258
+ if not verify['readable']:
4259
+ print(f'VERIFY FAILED: index still unreadable — {verify["error"]}')
4260
+ return 1
4261
+ bad = [e for e in verify['entries'] if e['status'] != 'ok']
4262
+ print(f'verify : index readable, {len(verify["entries"])} entries, {len(bad)} dangling')
4263
+ for entry in bad:
4264
+ print(f' still dangling: {entry["name"]!r} — triage with inspect')
4265
+ return 0
4266
+
4267
+ if args.cmd == 'inspect':
4268
+ with open_volume(args.device, readonly=True) as volume:
4269
+ verdict = volume.classify_dirent(args.dir, args.name)
4270
+ print(f'entry : {verdict["name"]!r} in {args.dir!r} '
4271
+ f'-> mft={verdict["dirent_record"]} seq={verdict["dirent_seq"]}')
4272
+ record = verdict.get('record_info')
4273
+ if record and record.get('open'):
4274
+ print(f'record : seq={record["seq"]} in_use={record["in_use"]} '
4275
+ f'links={record["link_count"]} dir={record["is_dir"]}')
4276
+ for entry in record['names']:
4277
+ print(f' claims : {entry["name"]!r} (parent mft={entry["parent_record"]})')
4278
+ elif record:
4279
+ print(f'record : unreadable/free ({record["error"]})')
4280
+ print(f'state : {verdict["state"]}')
4281
+ print(f'action : {verdict["action"]}')
4282
+ return 0 if verdict['state'] == 'healthy' else 1
4283
+
4284
+ if args.cmd == 'purge':
4285
+ with open_volume(args.device, readonly=not args.really) as volume:
4286
+ verdict = volume.purge_orphan(args.dir, args.name, really=args.really)
4287
+ if args.really:
4288
+ print(f'purged {args.name!r}: dirent removed, record '
4289
+ f'{verdict["dirent_record"]} and its clusters freed')
4290
+ else:
4291
+ print(f'DRY RUN: {args.name!r} is a true orphan (record {verdict["dirent_record"]}); '
4292
+ 're-run with --really to reclaim it fully')
4293
+ return 0
4294
+
4295
+ if args.cmd == 'scan':
4296
+ with open_volume(args.device, readonly=True) as volume:
4297
+ try:
4298
+ entries = volume.scan_dir(args.dir)
4299
+ except NtfsError as exc:
4300
+ print(f'index walk failed: {exc}')
4301
+ print('the directory index itself is unreadable (torn INDX block?) — '
4302
+ 'triage with `rebuild` (dry-run first)')
4303
+ return 2
4304
+ broken = 0
4305
+ for entry in entries:
4306
+ if entry['status'] == 'dir-self':
4307
+ continue
4308
+ flag = ' ' if entry['status'] == 'ok' else '!'
4309
+ print(f'{flag} {entry["name"]!r:60} mft={entry["record"]} seq={entry["seq"]} {entry["status"]}')
4310
+ broken += entry['status'] != 'ok'
4311
+ print(f'-- {broken} dangling entr{"y" if broken == 1 else "ies"}')
4312
+ return 1 if broken else 0
4313
+
4314
+ if not args.really:
4315
+ with open_volume(args.device, readonly=True) as volume:
4316
+ found = volume.remove_dirent(args.dir, args.name, really=False)
4317
+ print(f'DRY RUN: would remove {found["name"]!r} -> mft={found["record"]} seq={found["seq"]}')
4318
+ print('re-run with --really to write')
4319
+ return 0
4320
+
4321
+ with open_volume(args.device, readonly=False) as volume:
4322
+ found = volume.remove_dirent(args.dir, args.name, really=True)
4323
+ print(f'removed {found["name"]!r} (was mft={found["record"]} seq={found["seq"]})')
4324
+ return 0
4325
+
4326
+
4327
+ if __name__ == '__main__':
4328
+ sys.exit(main())