checkdisk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checkdisk.py
ADDED
|
@@ -0,0 +1,4328 @@
|
|
|
1
|
+
'''A self-contained, pure-Python NTFS repair tool — the parts of chkdsk /f that
|
|
2
|
+
matter for a crashed volume, with no dependency beyond the standard library.
|
|
3
|
+
|
|
4
|
+
A crash can leave a directory's B+ tree ($I30 index) holding entries whose MFT
|
|
5
|
+
record is dead or reused (sequence-number mismatch). The ntfs3 kernel driver then
|
|
6
|
+
returns ESTALE/EINVAL on those names forever, and `ntfsfix` cannot repair indexes.
|
|
7
|
+
Windows chkdsk fixes this in stage 2 by validating each index entry against the
|
|
8
|
+
MFT and deleting the dangling ones. This tool does that and the other repairs
|
|
9
|
+
below with its own native read/write engine: it parses and rewrites the on-disk
|
|
10
|
+
NTFS structures directly (records, runlists, B+ tree splits/merges, $Bitmap,
|
|
11
|
+
$Secure, the USN journal), with multi-sector fixups and plan-then-commit
|
|
12
|
+
atomicity.
|
|
13
|
+
|
|
14
|
+
Commands (device = partition or image file; dir = path INSIDE the volume):
|
|
15
|
+
scan <device> <dir> read-only: every entry health-checked
|
|
16
|
+
inspect <device> <dir> <name> read-only triage: free/reused/orphan?
|
|
17
|
+
rm <device> <dir> <name> [--really] drop one dangling entry
|
|
18
|
+
purge <device> <dir> <name> [--really] true orphan: free dirent+record+clusters
|
|
19
|
+
rebuild <device> <dir> [--really] torn INDX: rebuild the whole index
|
|
20
|
+
from the children's MFT records
|
|
21
|
+
fix|/f <device> [--really] chkdsk /f flow — stage 1: verify MFT
|
|
22
|
+
records + attribute runlists (repairs
|
|
23
|
+
torn empty-attribute truncates);
|
|
24
|
+
stage 2: walk every directory index,
|
|
25
|
+
repair dangling entries, rebuild torn
|
|
26
|
+
indexes; stage 5: reconcile $Bitmap
|
|
27
|
+
cluster accounting
|
|
28
|
+
backup <device> <outdir> [dir ...] dump boot sectors, $MFT, $MFTMirr and
|
|
29
|
+
the given directories' raw $I30 indexes
|
|
30
|
+
(targeted pre-repair backup)
|
|
31
|
+
usn <device> [--really] [--reset] validate the USN change journal
|
|
32
|
+
($UsnJrnl: $Max sanity + full $J record
|
|
33
|
+
walk); --really resets a corrupt
|
|
34
|
+
journal, --reset forces the reset
|
|
35
|
+
secure <device> [--really] chkdsk stage 3: validate $Secure —
|
|
36
|
+
every $SDS descriptor (header, hash,
|
|
37
|
+
256KiB mirror), the $SII/$SDH indexes
|
|
38
|
+
against $SDS, and every file's
|
|
39
|
+
security_id; --really repairs mirrors
|
|
40
|
+
and rebuilds the indexes
|
|
41
|
+
surface <device> chkdsk stage 4, report-only: read every
|
|
42
|
+
cluster, attribute unreadable ones to
|
|
43
|
+
their owning files (also available as
|
|
44
|
+
`/r` or `fix --surface`)
|
|
45
|
+
All writing commands are dry-run unless --really is provided.
|
|
46
|
+
'''
|
|
47
|
+
|
|
48
|
+
from __future__ import annotations
|
|
49
|
+
|
|
50
|
+
import argparse, array, bisect, errno, hashlib, os, struct, subprocess, sys, time
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
__version__ = '0.1.0'
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# Positioned I/O. os.pread/os.pwrite are POSIX-only; on Windows fall back to
|
|
57
|
+
# lseek + read/write — the tool is single-threaded, so the atomicity a real
|
|
58
|
+
# pread would give over a separate seek is moot here. _O_BINARY keeps Windows
|
|
59
|
+
# from CRLF/EOF-translating a raw disk image (it is 0, a no-op, on POSIX).
|
|
60
|
+
_O_BINARY = getattr(os, 'O_BINARY', 0)
|
|
61
|
+
if hasattr(os, 'pread'):
|
|
62
|
+
_pread, _pwrite = os.pread, os.pwrite
|
|
63
|
+
else:
|
|
64
|
+
def _pread(fd, n, offset):
|
|
65
|
+
os.lseek(fd, offset, os.SEEK_SET)
|
|
66
|
+
return os.read(fd, n)
|
|
67
|
+
|
|
68
|
+
def _pwrite(fd, data, offset):
|
|
69
|
+
os.lseek(fd, offset, os.SEEK_SET)
|
|
70
|
+
return os.write(fd, data)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
MREF_MASK = (1 << 48) - 1 # low 48 bits = record number, high 16 = sequence
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# UTF-16LE attribute/stream names (plain bytes; the reader matches names
|
|
77
|
+
# as bytes).
|
|
78
|
+
INDEX_I30 = '$I30'.encode('utf-16-le')
|
|
79
|
+
USN_MAX_NAME = '$Max'.encode('utf-16-le')
|
|
80
|
+
USN_J_NAME = '$J'.encode('utf-16-le')
|
|
81
|
+
SDS_NAME = '$SDS'.encode('utf-16-le')
|
|
82
|
+
SII_NAME = '$SII'.encode('utf-16-le')
|
|
83
|
+
SDH_NAME = '$SDH'.encode('utf-16-le')
|
|
84
|
+
R_NAME = '$R'.encode('utf-16-le')
|
|
85
|
+
O_NAME = '$O'.encode('utf-16-le')
|
|
86
|
+
|
|
87
|
+
SDS_BLOCK = 0x40000 # $SDS entries mirror in alternating 256 KiB blocks
|
|
88
|
+
|
|
89
|
+
AT_FILE_NAME = 0x30
|
|
90
|
+
AT_DATA = 0x80
|
|
91
|
+
AT_INDEX_ROOT = 0x90
|
|
92
|
+
AT_INDEX_ALLOCATION = 0xA0
|
|
93
|
+
AT_BITMAP = 0xB0
|
|
94
|
+
AT_STANDARD_INFORMATION = 0x10
|
|
95
|
+
AT_ATTRIBUTE_LIST = 0x20
|
|
96
|
+
FILE_ROOT = 5
|
|
97
|
+
FILE_BITMAP = 6
|
|
98
|
+
FILE_LOGFILE = 2
|
|
99
|
+
FILE_SECURE = 9
|
|
100
|
+
FILE_UPCASE = 10
|
|
101
|
+
|
|
102
|
+
AT_UNNAMED = None
|
|
103
|
+
|
|
104
|
+
NTFS_DT_DIR = 4 # readdir dt_type for a directory (mirrors DT_DIR)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class NtfsError(OSError):
|
|
108
|
+
pass
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# ── raw MFT record helpers (read-only parsing for the rebuild census) ────────
|
|
112
|
+
|
|
113
|
+
def _apply_fixups(rec: bytearray) -> bool:
|
|
114
|
+
'''Undo the multi-sector transfer protection; False = the record is torn '''
|
|
115
|
+
|
|
116
|
+
usa_ofs = struct.unpack_from('<H', rec, 4)[0]
|
|
117
|
+
usa_count = struct.unpack_from('<H', rec, 6)[0]
|
|
118
|
+
if not usa_ofs or usa_ofs + 2 * usa_count > len(rec):
|
|
119
|
+
return False
|
|
120
|
+
usn = bytes(rec[usa_ofs:usa_ofs + 2])
|
|
121
|
+
for i in range(1, usa_count):
|
|
122
|
+
end = i * 512
|
|
123
|
+
if end > len(rec) or bytes(rec[end - 2:end]) != usn:
|
|
124
|
+
return False
|
|
125
|
+
rec[end - 2:end] = rec[usa_ofs + 2 * i:usa_ofs + 2 * i + 2]
|
|
126
|
+
return True
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _attrs(rec):
|
|
130
|
+
'''Yield (offset, attr_type, attr_len) along a fixed-up record's attribute
|
|
131
|
+
chain, stopping silently at the 0xFFFFFFFF terminator or any malformed
|
|
132
|
+
header — the shared walk. The one caller that must REPORT a broken
|
|
133
|
+
chain rather than stop (mft_survey) keeps its own loop.'''
|
|
134
|
+
offset = struct.unpack_from('<H', rec, 20)[0]
|
|
135
|
+
while offset + 8 <= len(rec):
|
|
136
|
+
attr_type = struct.unpack_from('<I', rec, offset)[0]
|
|
137
|
+
if attr_type == 0xFFFFFFFF:
|
|
138
|
+
return
|
|
139
|
+
length = struct.unpack_from('<I', rec, offset + 4)[0]
|
|
140
|
+
if length < 24 or offset + length > len(rec):
|
|
141
|
+
return
|
|
142
|
+
yield offset, attr_type, length
|
|
143
|
+
offset += length
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _has_attr(rec: bytes, want: int) -> bool:
|
|
147
|
+
'''True if the fixed-up MFT record carries an attribute of this type '''
|
|
148
|
+
return any(t == want for _off, t, _len in _attrs(rec))
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _seal_fixups(rec: bytearray) -> None:
|
|
152
|
+
'''Inverse of _apply_fixups: bump the update sequence number, stash each
|
|
153
|
+
sector's real last word into the USA, and stamp the USN over the
|
|
154
|
+
sector ends — the multi-sector transfer protection, applied.'''
|
|
155
|
+
usa_ofs = struct.unpack_from('<H', rec, 4)[0]
|
|
156
|
+
usa_count = struct.unpack_from('<H', rec, 6)[0]
|
|
157
|
+
usn = (struct.unpack_from('<H', rec, usa_ofs)[0] + 1) & 0xFFFF or 1
|
|
158
|
+
struct.pack_into('<H', rec, usa_ofs, usn)
|
|
159
|
+
for i in range(1, usa_count):
|
|
160
|
+
end = i * 512
|
|
161
|
+
rec[usa_ofs + 2 * i:usa_ofs + 2 * i + 2] = rec[end - 2:end]
|
|
162
|
+
struct.pack_into('<H', rec, end - 2, usn)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _file_name_attrs(rec: bytes):
|
|
166
|
+
'''Yield each resident $FILE_NAME attribute value in a fixed-up MFT record '''
|
|
167
|
+
for offset, attr_type, _length in _attrs(rec):
|
|
168
|
+
if attr_type == AT_FILE_NAME and rec[offset + 8] == 0: # resident only
|
|
169
|
+
value_len = struct.unpack_from('<I', rec, offset + 16)[0]
|
|
170
|
+
value_ofs = struct.unpack_from('<H', rec, offset + 20)[0]
|
|
171
|
+
fn = rec[offset + value_ofs:offset + value_ofs + value_len]
|
|
172
|
+
if len(fn) >= 66 and len(fn) >= 66 + 2 * fn[64]:
|
|
173
|
+
yield fn
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _check_mapping_pairs(rec: bytes, attr_off: int, length: int,
|
|
177
|
+
nr_clusters: int) -> tuple[str | None, bool, list]:
|
|
178
|
+
'''Validate and decode a non-resident attribute's mapping pairs: sane pair
|
|
179
|
+
headers, no overrun, positive run lengths,
|
|
180
|
+
LCNs inside the volume, total clusters matching the highest_vcn header.
|
|
181
|
+
Returns (problem, is_phantom_empty, runs) — runs is the decoded
|
|
182
|
+
[(lcn, run_len, vcn), ...] for allocated (non-sparse) runs, the shared
|
|
183
|
+
source for the stage-5 used-cluster map; is_phantom_empty marks the one
|
|
184
|
+
safely auto-repairable case: a header that says "empty" (highest_vcn
|
|
185
|
+
== -1) while well-formed pairs still describe runs (torn truncate).'''
|
|
186
|
+
|
|
187
|
+
lowest = struct.unpack_from('<q', rec, attr_off + 16)[0]
|
|
188
|
+
highest = struct.unpack_from('<q', rec, attr_off + 24)[0]
|
|
189
|
+
mp_ofs = struct.unpack_from('<H', rec, attr_off + 32)[0]
|
|
190
|
+
end = attr_off + length
|
|
191
|
+
pos = attr_off + mp_ofs
|
|
192
|
+
out: list[tuple[int, int]] = []
|
|
193
|
+
if pos >= end:
|
|
194
|
+
return f'mapping pairs offset {mp_ofs} outside attribute', False, out
|
|
195
|
+
clusters, lcn, vcn = 0, 0, lowest
|
|
196
|
+
while pos < end and rec[pos]:
|
|
197
|
+
header = rec[pos]
|
|
198
|
+
len_sz, ofs_sz = header & 0xF, header >> 4
|
|
199
|
+
if not 1 <= len_sz <= 8 or ofs_sz > 8:
|
|
200
|
+
return f'bad pair header {header:#x} at +{pos - attr_off}', False, out
|
|
201
|
+
if pos + 1 + len_sz + ofs_sz > end:
|
|
202
|
+
return f'pair overruns attribute at +{pos - attr_off}', False, out
|
|
203
|
+
run_len = int.from_bytes(rec[pos + 1:pos + 1 + len_sz], 'little',
|
|
204
|
+
signed=True)
|
|
205
|
+
if run_len <= 0:
|
|
206
|
+
return f'non-positive run length {run_len} at +{pos - attr_off}', False, out
|
|
207
|
+
if ofs_sz: # 0 = sparse run, lcn unchanged
|
|
208
|
+
lcn += int.from_bytes(rec[pos + 1 + len_sz:pos + 1 + len_sz + ofs_sz],
|
|
209
|
+
'little', signed=True)
|
|
210
|
+
if lcn < 0:
|
|
211
|
+
return f'negative LCN {lcn} at +{pos - attr_off}', False, out
|
|
212
|
+
if lcn + run_len > nr_clusters:
|
|
213
|
+
return (f'run [{lcn}, {lcn + run_len}) beyond volume '
|
|
214
|
+
f'({nr_clusters} clusters)', False, out)
|
|
215
|
+
out.append((lcn, run_len, vcn))
|
|
216
|
+
clusters += run_len
|
|
217
|
+
vcn += run_len
|
|
218
|
+
pos += 1 + len_sz + ofs_sz
|
|
219
|
+
if pos >= end:
|
|
220
|
+
return 'runlist not terminated inside attribute', False, out
|
|
221
|
+
if highest == -1 and lowest == 0:
|
|
222
|
+
if clusters:
|
|
223
|
+
return (f'{len(out) or 1} phantom run(s) on an empty attribute '
|
|
224
|
+
'(torn truncate)', True, out)
|
|
225
|
+
elif highest >= lowest >= 0 and clusters != highest - lowest + 1:
|
|
226
|
+
return (f'runlist covers {clusters} clusters, '
|
|
227
|
+
f'header says {highest - lowest + 1}'), False, out
|
|
228
|
+
return None, False, out
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _min_bytes_signed(v: int) -> bytes:
|
|
232
|
+
n = 1
|
|
233
|
+
while not -(1 << (8 * n - 1)) <= v < (1 << (8 * n - 1)):
|
|
234
|
+
n += 1
|
|
235
|
+
return v.to_bytes(n, 'little', signed=True)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _encode_mapping_pairs(runs) -> bytes:
|
|
239
|
+
'''Encode VCN-contiguous [(lcn, length), ...] into NTFS mapping pairs —
|
|
240
|
+
the inverse of _check_mapping_pairs' decode. lcn < 0 marks a sparse run
|
|
241
|
+
(the offset field is omitted). Terminated by a 0 header byte.'''
|
|
242
|
+
out = bytearray()
|
|
243
|
+
prev = 0
|
|
244
|
+
for lcn, length in runs:
|
|
245
|
+
if length <= 0:
|
|
246
|
+
raise NtfsError(0, f'non-positive run length {length}')
|
|
247
|
+
# both fields are signed varints: a length of 128 must be encoded as
|
|
248
|
+
# 80 00 (two bytes) — the driver sign-extends the top byte and treats
|
|
249
|
+
# a lone 80 as -128, refusing the whole runlist
|
|
250
|
+
len_bytes = _min_bytes_signed(length)
|
|
251
|
+
if lcn < 0: # sparse
|
|
252
|
+
out.append(len(len_bytes))
|
|
253
|
+
out += len_bytes
|
|
254
|
+
else:
|
|
255
|
+
off_bytes = _min_bytes_signed(lcn - prev)
|
|
256
|
+
out.append((len(off_bytes) << 4) | len(len_bytes))
|
|
257
|
+
out += len_bytes + off_bytes
|
|
258
|
+
prev = lcn
|
|
259
|
+
out.append(0)
|
|
260
|
+
return bytes(out)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _walk_usn_records(read, start: int, end: int) -> tuple[int, list[str]]:
|
|
264
|
+
'''Validate the USN $J stream in [start, end): every record's length is
|
|
265
|
+
sane and 8-aligned, its Usn field equals its own byte offset, names stay
|
|
266
|
+
inside the record, and zero fill appears only as tail padding up to the
|
|
267
|
+
next 4 KiB page boundary. `read(pos, count) -> bytes`. Returns
|
|
268
|
+
(records_walked, problems); stops at the first structural problem — a
|
|
269
|
+
corrupt journal gets reset wholesale, so one finding is enough.'''
|
|
270
|
+
|
|
271
|
+
problems: list[str] = []
|
|
272
|
+
count, pos = 0, start
|
|
273
|
+
if pos & 7:
|
|
274
|
+
return 0, [f'journal start {pos} not 8-byte aligned']
|
|
275
|
+
while pos < end and not problems:
|
|
276
|
+
chunk = read(pos, min(1 << 20, end - pos))
|
|
277
|
+
if not chunk:
|
|
278
|
+
problems.append(f'$J read failed at usn {pos}')
|
|
279
|
+
break
|
|
280
|
+
off, n = 0, len(chunk)
|
|
281
|
+
while off + 4 <= n and not problems:
|
|
282
|
+
reclen = struct.unpack_from('<I', chunk, off)[0]
|
|
283
|
+
if reclen == 0:
|
|
284
|
+
# zero fill is only valid up to the next 4 KiB page boundary
|
|
285
|
+
nxt = min((pos + off + 4096) & ~4095, end) - pos
|
|
286
|
+
if chunk[off:min(nxt, n)].strip(b'\0'):
|
|
287
|
+
problems.append(f'garbage inside zero padding at usn {pos + off}')
|
|
288
|
+
off = min(nxt, n)
|
|
289
|
+
continue
|
|
290
|
+
if reclen & 7 or not 60 <= reclen <= 8192:
|
|
291
|
+
problems.append(f'bad record length {reclen} at usn {pos + off}')
|
|
292
|
+
break
|
|
293
|
+
if off + reclen > n:
|
|
294
|
+
break # record crosses the chunk edge — refill from here
|
|
295
|
+
major = struct.unpack_from('<H', chunk, off + 4)[0]
|
|
296
|
+
if major not in (2, 3, 4):
|
|
297
|
+
problems.append(f'unknown USN record version {major} at usn {pos + off}')
|
|
298
|
+
break
|
|
299
|
+
usn_field = {2: 24, 3: 40}.get(major) # v4 carries no Usn to cross-check
|
|
300
|
+
if usn_field is not None:
|
|
301
|
+
claimed = struct.unpack_from('<q', chunk, off + usn_field)[0]
|
|
302
|
+
if claimed != pos + off:
|
|
303
|
+
problems.append(f'record at offset {pos + off} claims usn {claimed}')
|
|
304
|
+
break
|
|
305
|
+
if major == 2:
|
|
306
|
+
nlen, nofs = struct.unpack_from('<HH', chunk, off + 56)
|
|
307
|
+
if nofs + nlen > reclen:
|
|
308
|
+
problems.append(f'file name overruns record at usn {pos + off}')
|
|
309
|
+
break
|
|
310
|
+
count += 1
|
|
311
|
+
off += reclen
|
|
312
|
+
if problems:
|
|
313
|
+
break
|
|
314
|
+
if off == 0:
|
|
315
|
+
problems.append(f'truncated record at usn {pos} (overruns journal end)')
|
|
316
|
+
break
|
|
317
|
+
pos += off
|
|
318
|
+
return count, problems
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _security_hash(descriptor: bytes) -> int:
|
|
322
|
+
'''The $Secure descriptor hash: over each little-endian u32 word,
|
|
323
|
+
hash = word + rol32(hash, 3). Trailing 1-3 bytes are ignored.'''
|
|
324
|
+
h = 0
|
|
325
|
+
for (word,) in struct.iter_unpack('<I', descriptor[:len(descriptor) & ~3]):
|
|
326
|
+
h = (word + ((h << 3) | (h >> 29))) & 0xFFFFFFFF
|
|
327
|
+
return h
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _parse_sds(sds: bytes) -> tuple[dict, list[str], list[tuple]]:
|
|
331
|
+
'''Walk $SDS: entries live in even 256 KiB blocks, each mirrored into the
|
|
332
|
+
following odd block. Returns ({security_id: (hash, offset, length)},
|
|
333
|
+
problems, mirror_fixes) where each mirror fix is (offset, length, source)
|
|
334
|
+
with source 'primary' or 'mirror' — the side whose hash verifies.'''
|
|
335
|
+
|
|
336
|
+
entries: dict[int, tuple] = {}
|
|
337
|
+
problems: list[str] = []
|
|
338
|
+
fixes: list[tuple] = []
|
|
339
|
+
pos, end = 0, len(sds)
|
|
340
|
+
while pos + 20 <= end:
|
|
341
|
+
block = pos // SDS_BLOCK
|
|
342
|
+
if block & 1: # mirror region — validated alongside its primary
|
|
343
|
+
pos = (block + 1) * SDS_BLOCK
|
|
344
|
+
continue
|
|
345
|
+
hash_, sid, off, length = struct.unpack_from('<IIQI', sds, pos)
|
|
346
|
+
if length == 0: # end of entries in this block
|
|
347
|
+
pos = (block + 2) * SDS_BLOCK
|
|
348
|
+
continue
|
|
349
|
+
if length < 20 or pos + length > min(end, (block + 1) * SDS_BLOCK):
|
|
350
|
+
problems.append(f'bad $SDS entry length {length} at offset {pos}')
|
|
351
|
+
break
|
|
352
|
+
if off != pos:
|
|
353
|
+
problems.append(f'$SDS entry at {pos} claims offset {off}')
|
|
354
|
+
break
|
|
355
|
+
primary_ok = _security_hash(sds[pos + 20:pos + length]) == hash_
|
|
356
|
+
mirror_pos = pos + SDS_BLOCK
|
|
357
|
+
mirror_ok = None
|
|
358
|
+
if mirror_pos + length <= end:
|
|
359
|
+
mirror = sds[mirror_pos:mirror_pos + length]
|
|
360
|
+
# a faithful mirror matches byte-for-byte; tolerate an offset field
|
|
361
|
+
# that points at itself instead of the primary
|
|
362
|
+
same = (mirror[:8] == sds[pos:pos + 8]
|
|
363
|
+
and mirror[16:] == sds[pos + 16:pos + length]
|
|
364
|
+
and struct.unpack_from('<Q', mirror, 8)[0] in (pos, mirror_pos))
|
|
365
|
+
if same:
|
|
366
|
+
mirror_ok = True
|
|
367
|
+
else:
|
|
368
|
+
mhash = struct.unpack_from('<I', mirror, 0)[0]
|
|
369
|
+
mirror_ok = (_security_hash(mirror[20:]) == mhash
|
|
370
|
+
and struct.unpack_from('<I', mirror, 4)[0] == sid)
|
|
371
|
+
if primary_ok:
|
|
372
|
+
fixes.append((pos, length, 'primary'))
|
|
373
|
+
elif mirror_ok:
|
|
374
|
+
fixes.append((pos, length, 'mirror'))
|
|
375
|
+
else:
|
|
376
|
+
problems.append(f'$SDS entry id {sid} at {pos}: both copies corrupt')
|
|
377
|
+
if not primary_ok and mirror_ok is not True:
|
|
378
|
+
if mirror_ok is None:
|
|
379
|
+
problems.append(f'$SDS entry id {sid} at {pos}: hash mismatch, no mirror')
|
|
380
|
+
if sid in entries:
|
|
381
|
+
problems.append(f'duplicate security id {sid} in $SDS')
|
|
382
|
+
entries[sid] = (hash_, pos, length)
|
|
383
|
+
pos = (pos + length + 15) & ~15
|
|
384
|
+
return entries, problems, fixes
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _walk_index_node(buf: bytes, hdr_off: int, entries: list, problems: list,
|
|
388
|
+
label: str, dir_index: bool = False, key_fn=None) -> None:
|
|
389
|
+
'''Collect (key, data) from one index node whose INDEX_HEADER sits at
|
|
390
|
+
hdr_off — works for $INDEX_ROOT values and fixed-up INDX blocks alike.'''
|
|
391
|
+
if hdr_off + 16 > len(buf):
|
|
392
|
+
problems.append(f'{label}: truncated index header')
|
|
393
|
+
return
|
|
394
|
+
prev = None
|
|
395
|
+
entries_ofs, index_len = struct.unpack_from('<II', buf, hdr_off)
|
|
396
|
+
pos = hdr_off + entries_ofs
|
|
397
|
+
limit = min(hdr_off + index_len, len(buf))
|
|
398
|
+
while pos + 16 <= limit:
|
|
399
|
+
data_ofs, data_len = struct.unpack_from('<HH', buf, pos)
|
|
400
|
+
length, key_len, flags = struct.unpack_from('<HHH', buf, pos + 8)
|
|
401
|
+
if length < 16 or pos + length > limit:
|
|
402
|
+
problems.append(f'{label}: bad index entry length {length} at +{pos}')
|
|
403
|
+
return
|
|
404
|
+
if not flags & 2: # not the END entry
|
|
405
|
+
if key_fn is not None and 16 + key_len <= length:
|
|
406
|
+
cur = key_fn(buf[pos + 16:pos + 16 + key_len])
|
|
407
|
+
if prev is not None and cur < prev:
|
|
408
|
+
problems.append(f'{label}: keys out of order at +{pos}')
|
|
409
|
+
prev = cur
|
|
410
|
+
if dir_index:
|
|
411
|
+
if 16 + key_len > length:
|
|
412
|
+
problems.append(f'{label}: entry bounds corrupt at +{pos}')
|
|
413
|
+
return
|
|
414
|
+
entries.append((buf[pos + 16:pos + 16 + key_len],
|
|
415
|
+
struct.unpack_from('<Q', buf, pos)[0]))
|
|
416
|
+
else:
|
|
417
|
+
if 16 + key_len > length or data_ofs + data_len > length:
|
|
418
|
+
problems.append(f'{label}: entry bounds corrupt at +{pos}')
|
|
419
|
+
return
|
|
420
|
+
entries.append((buf[pos + 16:pos + 16 + key_len],
|
|
421
|
+
buf[pos + data_ofs:pos + data_ofs + data_len]))
|
|
422
|
+
if flags & 2:
|
|
423
|
+
return
|
|
424
|
+
pos += length
|
|
425
|
+
problems.append(f'{label}: node not terminated by an END entry')
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _view_index_entries(root: bytes, alloc: bytes | None, bitmap: bytes | None,
|
|
429
|
+
label: str, dir_index: bool = False,
|
|
430
|
+
key_fn=None) -> tuple[list, list]:
|
|
431
|
+
'''All (key, data) pairs of a view index: the $INDEX_ROOT node plus every
|
|
432
|
+
in-use, fixup-verified INDX block of $INDEX_ALLOCATION.'''
|
|
433
|
+
entries: list[tuple[bytes, bytes]] = []
|
|
434
|
+
problems: list[str] = []
|
|
435
|
+
if len(root) < 32:
|
|
436
|
+
return [], [f'{label}: $INDEX_ROOT too small ({len(root)} bytes)']
|
|
437
|
+
_walk_index_node(root, 16, entries, problems, f'{label} root', dir_index, key_fn)
|
|
438
|
+
if alloc:
|
|
439
|
+
block_size = struct.unpack_from('<I', root, 8)[0]
|
|
440
|
+
if block_size not in (512, 1024, 2048, 4096, 8192):
|
|
441
|
+
return entries, [f'{label}: implausible index block size {block_size}']
|
|
442
|
+
for i in range(len(alloc) // block_size):
|
|
443
|
+
if bitmap is not None and not (i >> 3) < len(bitmap):
|
|
444
|
+
break
|
|
445
|
+
if bitmap is not None and not bitmap[i >> 3] & (1 << (i & 7)):
|
|
446
|
+
continue # free block — stale content is fine
|
|
447
|
+
blk = bytearray(alloc[i * block_size:(i + 1) * block_size])
|
|
448
|
+
if blk[:4] != b'INDX':
|
|
449
|
+
problems.append(f'{label} block {i}: bad magic')
|
|
450
|
+
continue
|
|
451
|
+
if not _apply_fixups(blk):
|
|
452
|
+
problems.append(f'{label} block {i}: torn (fixup mismatch)')
|
|
453
|
+
continue
|
|
454
|
+
_walk_index_node(bytes(blk), 24, entries, problems,
|
|
455
|
+
f'{label} block {i}', dir_index, key_fn)
|
|
456
|
+
return entries, problems
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _surface_scan(pread, end: int, cluster_size: int, progress=None) -> list[int]:
|
|
460
|
+
'''Read [0, end) sequentially in large chunks; when a chunk read fails,
|
|
461
|
+
bisect it down to single clusters. pread(pos, count) -> bytes and raises
|
|
462
|
+
OSError on unreadable media. Returns sorted unreadable cluster numbers.'''
|
|
463
|
+
|
|
464
|
+
bad: list[int] = []
|
|
465
|
+
|
|
466
|
+
def probe(pos: int, length: int) -> None:
|
|
467
|
+
if length <= cluster_size:
|
|
468
|
+
try:
|
|
469
|
+
pread(pos, length)
|
|
470
|
+
except OSError:
|
|
471
|
+
bad.append(pos // cluster_size)
|
|
472
|
+
return
|
|
473
|
+
half = (length // 2) // cluster_size * cluster_size or cluster_size
|
|
474
|
+
for p, n in ((pos, half), (pos + half, length - half)):
|
|
475
|
+
if n <= 0:
|
|
476
|
+
continue
|
|
477
|
+
try:
|
|
478
|
+
pread(p, n)
|
|
479
|
+
except OSError:
|
|
480
|
+
probe(p, n)
|
|
481
|
+
|
|
482
|
+
pos, chunk = 0, 64 << 20
|
|
483
|
+
while pos < end:
|
|
484
|
+
want = min(chunk, end - pos)
|
|
485
|
+
try:
|
|
486
|
+
got = len(pread(pos, want))
|
|
487
|
+
except OSError:
|
|
488
|
+
probe(pos, want)
|
|
489
|
+
pos += want
|
|
490
|
+
got = None
|
|
491
|
+
if got is not None:
|
|
492
|
+
if got == 0: # outside the try: NtfsError subclasses OSError
|
|
493
|
+
raise NtfsError(0, f'device ends prematurely at byte {pos}')
|
|
494
|
+
pos += got
|
|
495
|
+
if progress:
|
|
496
|
+
progress(pos, end)
|
|
497
|
+
return sorted(set(bad))
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def _set_run(bitmap: bytearray, lcn: int, length: int) -> None:
|
|
501
|
+
'''Mark clusters [lcn, lcn+length) used in an LSB-first bitmap.'''
|
|
502
|
+
end = lcn + length
|
|
503
|
+
first_full = (lcn + 7) // 8
|
|
504
|
+
last_full = end // 8
|
|
505
|
+
for c in range(lcn, min(first_full * 8, end)):
|
|
506
|
+
bitmap[c >> 3] |= 1 << (c & 7)
|
|
507
|
+
if last_full > first_full:
|
|
508
|
+
bitmap[first_full:last_full] = b'\xff' * (last_full - first_full)
|
|
509
|
+
for c in range(max(last_full * 8, lcn), end):
|
|
510
|
+
bitmap[c >> 3] |= 1 << (c & 7)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
# ── volume handling ──────────────────────────────────────────────────────────
|
|
514
|
+
|
|
515
|
+
def _mount_table(mounts: str) -> list[tuple[str, str]]:
|
|
516
|
+
'''(source, mountpoint) pairs — /proc/mounts where it exists, the mount(8)
|
|
517
|
+
"SRC on MNT (opts)" output on macOS/FreeBSD.'''
|
|
518
|
+
if os.path.exists(mounts):
|
|
519
|
+
with open(mounts) as fh:
|
|
520
|
+
return [tuple(line.split()[:2]) for line in fh]
|
|
521
|
+
out = subprocess.run(['mount'], capture_output=True, text=True).stdout
|
|
522
|
+
pairs = []
|
|
523
|
+
for line in out.splitlines():
|
|
524
|
+
if ' on ' in line:
|
|
525
|
+
src, _, rest = line.partition(' on ')
|
|
526
|
+
mnt = rest.rsplit(' (', 1)[0].split(' type ')[0]
|
|
527
|
+
pairs.append((src.strip(), mnt.strip()))
|
|
528
|
+
return pairs
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def assert_not_mounted(device: str, mounts: str = '/proc/mounts',
|
|
532
|
+
sys_block: str = '/sys/class/block') -> None:
|
|
533
|
+
real = os.path.realpath(device)
|
|
534
|
+
for src, mnt in _mount_table(mounts):
|
|
535
|
+
if not src.startswith('/'):
|
|
536
|
+
continue
|
|
537
|
+
src_real = os.path.realpath(src)
|
|
538
|
+
if src_real == real:
|
|
539
|
+
raise SystemExit(f'{device} is mounted ({mnt}) — unmount it first')
|
|
540
|
+
# An image file attached to a loop device mounts as /dev/loopN (or
|
|
541
|
+
# a partition /dev/loopNpM); the origin is only visible in the
|
|
542
|
+
# loop's sysfs backing_file. The ../ variant reaches the parent
|
|
543
|
+
# loop device from a partition's sysfs node.
|
|
544
|
+
name = os.path.basename(src_real)
|
|
545
|
+
for backing in (f'{sys_block}/{name}/loop/backing_file',
|
|
546
|
+
f'{sys_block}/{name}/../loop/backing_file'):
|
|
547
|
+
try:
|
|
548
|
+
with open(backing) as bf:
|
|
549
|
+
backing_real = os.path.realpath(bf.read().strip())
|
|
550
|
+
except OSError:
|
|
551
|
+
continue
|
|
552
|
+
if backing_real == real:
|
|
553
|
+
raise SystemExit(f'{device} is mounted via {src_real} on {mnt} '
|
|
554
|
+
'— detach/unmount it first')
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
class RawVolume:
|
|
558
|
+
'''Read-only NTFS access: boot-sector bootstrap, $MFT runlist (including
|
|
559
|
+
$ATTRIBUTE_LIST-spanning $DATA), raw attribute and $I30 index walking,
|
|
560
|
+
path resolution, and all the analysis passes (MFT survey, cluster audit,
|
|
561
|
+
$Secure and USN validation). Pure Python — no external library. Every
|
|
562
|
+
write refuses here; the RawVolumeRW subclass adds the repair engine.'''
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def _invalidate_mft_cache(self) -> None:
|
|
566
|
+
'''Any repair may rewrite MFT records — drop the raw-read cache.'''
|
|
567
|
+
getattr(self, '_mft_chunk_cache', {}).clear()
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _load_record(self, rec_no: int) -> bytearray | None:
|
|
571
|
+
'''read_record + FILE magic + fixups — the standard triple. None when
|
|
572
|
+
the record is unreadable, not a FILE record, or torn.'''
|
|
573
|
+
raw = self.read_record(rec_no)
|
|
574
|
+
if raw is None or raw[:4] != b'FILE':
|
|
575
|
+
return None
|
|
576
|
+
rec = bytearray(raw)
|
|
577
|
+
return rec if _apply_fixups(rec) else None
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _mref_live(self, mref: int) -> bool:
|
|
581
|
+
'''True iff the referenced record is readable, in use, and still carries
|
|
582
|
+
the reference's sequence number (i.e. the reference is not stale).'''
|
|
583
|
+
rec = self._load_record(mref & MREF_MASK)
|
|
584
|
+
return (rec is not None
|
|
585
|
+
and bool(struct.unpack_from('<H', rec, 22)[0] & 1)
|
|
586
|
+
and struct.unpack_from('<H', rec, 16)[0] == mref >> 48)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def nr_clusters(self) -> int:
|
|
590
|
+
if getattr(self, '_nr_clusters', None) is None:
|
|
591
|
+
with open(self.device, 'rb') as fh:
|
|
592
|
+
boot = fh.read(72)
|
|
593
|
+
bps = struct.unpack_from('<H', boot, 11)[0]
|
|
594
|
+
spc = boot[13]
|
|
595
|
+
spc = 2 ** (256 - spc) if spc > 0x80 else spc
|
|
596
|
+
sectors = struct.unpack_from('<Q', boot, 40)[0]
|
|
597
|
+
self._cluster_size = bps * spc
|
|
598
|
+
self._nr_clusters = sectors // spc
|
|
599
|
+
return self._nr_clusters
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def cluster_size(self) -> int:
|
|
603
|
+
self.nr_clusters()
|
|
604
|
+
return self._cluster_size
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
# -- chkdsk stage-5: rebuild the true cluster map, diff against $Bitmap --
|
|
608
|
+
|
|
609
|
+
def cluster_audit(self, survey: dict | None = None) -> dict:
|
|
610
|
+
'''Rebuild the true used-cluster map and diff it against $Bitmap.
|
|
611
|
+
missing > 0 means live data sits on clusters marked free (danger);
|
|
612
|
+
extra > 0 is leaked space. `failures` non-empty means the map is
|
|
613
|
+
incomplete and MUST NOT be written back.
|
|
614
|
+
|
|
615
|
+
The map comes from the raw runlist decode of a single $MFT
|
|
616
|
+
pass (mft_survey, reusable across stages).'''
|
|
617
|
+
|
|
618
|
+
nc = self.nr_clusters()
|
|
619
|
+
s = survey if survey and survey.get('used') is not None \
|
|
620
|
+
else self.mft_survey(want_used=True)
|
|
621
|
+
return self._bitmap_diff(nc, s['used'], list(s['map_failures']))
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def _bitmap_diff(self, nc: int, used: bytearray, failures: list[str]) -> dict:
|
|
625
|
+
ondisk = self._read_whole_attr(FILE_BITMAP, AT_DATA)
|
|
626
|
+
n = len(used)
|
|
627
|
+
if len(ondisk) < n:
|
|
628
|
+
failures.append(f'$Bitmap shorter than expected ({len(ondisk)} < {n})')
|
|
629
|
+
ondisk = ondisk.ljust(n, b'\0')
|
|
630
|
+
mask = (1 << nc) - 1 # ignore padding bits past the last real cluster
|
|
631
|
+
u = int.from_bytes(bytes(used), 'little') & mask
|
|
632
|
+
d = int.from_bytes(ondisk[:n], 'little') & mask
|
|
633
|
+
extra, missing = d & ~u, u & ~d
|
|
634
|
+
return {'nr_clusters': nc, 'used_count': u.bit_count(),
|
|
635
|
+
'bitmap_count': d.bit_count(),
|
|
636
|
+
'extra': extra.bit_count(), 'missing': missing.bit_count(),
|
|
637
|
+
'used_bytes': bytes(used), 'ondisk_tail': ondisk[n - 1] if n else 0,
|
|
638
|
+
'failures': failures}
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
# -- chkdsk stage 3: $Secure ($SDS stream + $SII/$SDH indexes) --
|
|
642
|
+
|
|
643
|
+
def secure_check(self, survey: dict | None = None) -> dict:
|
|
644
|
+
'''Validate the security descriptor store: every $SDS entry's header,
|
|
645
|
+
hash and 256 KiB mirror; the $SII (id) and $SDH (hash) indexes agree
|
|
646
|
+
with $SDS entry-for-entry; and every in-use file's security_id
|
|
647
|
+
references an existing descriptor.'''
|
|
648
|
+
|
|
649
|
+
info = {'present': False, 'problems': [], 'notes': [], 'mirror_fixes': [],
|
|
650
|
+
'index_problems': [], 'ref_missing': [], 'sds_entries': {},
|
|
651
|
+
'descriptors': 0, 'files_checked': 0}
|
|
652
|
+
try:
|
|
653
|
+
sds = self._read_whole_attr(FILE_SECURE, AT_DATA, SDS_NAME, 4)
|
|
654
|
+
except NtfsError:
|
|
655
|
+
info['notes'].append('no $Secure/$SDS — pre-NTFS-3.0 style volume')
|
|
656
|
+
return info
|
|
657
|
+
info['present'] = True
|
|
658
|
+
|
|
659
|
+
entries, sds_problems, fixes = _parse_sds(sds)
|
|
660
|
+
info['sds_entries'] = entries
|
|
661
|
+
info['descriptors'] = len(entries)
|
|
662
|
+
info['problems'].extend(sds_problems)
|
|
663
|
+
info['mirror_fixes'] = fixes
|
|
664
|
+
|
|
665
|
+
for label, name, key_fmt in (('$SII', SII_NAME, '<I'), ('$SDH', SDH_NAME, '<II')):
|
|
666
|
+
try:
|
|
667
|
+
root = self._read_whole_attr(FILE_SECURE, AT_INDEX_ROOT, name, 4)
|
|
668
|
+
except NtfsError:
|
|
669
|
+
info['index_problems'].append(f'{label}: $INDEX_ROOT missing')
|
|
670
|
+
continue
|
|
671
|
+
alloc = bmp = None
|
|
672
|
+
try:
|
|
673
|
+
alloc = self._read_whole_attr(FILE_SECURE, AT_INDEX_ALLOCATION, name, 4)
|
|
674
|
+
bmp = self._read_whole_attr(FILE_SECURE, AT_BITMAP, name, 4)
|
|
675
|
+
except NtfsError:
|
|
676
|
+
pass # small index — resident root only
|
|
677
|
+
key_fn = (lambda k, f=key_fmt: struct.unpack_from(f, k)
|
|
678
|
+
if len(k) >= struct.calcsize(f) else ())
|
|
679
|
+
idx, idx_problems = _view_index_entries(root, alloc, bmp, label,
|
|
680
|
+
key_fn=key_fn)
|
|
681
|
+
info['index_problems'].extend(idx_problems)
|
|
682
|
+
seen = set()
|
|
683
|
+
for key, data in idx:
|
|
684
|
+
if len(data) < 20:
|
|
685
|
+
info['index_problems'].append(f'{label}: entry data too short')
|
|
686
|
+
continue
|
|
687
|
+
h, sid, off, length = struct.unpack_from('<IIQI', data, 0)
|
|
688
|
+
if label == '$SII':
|
|
689
|
+
kid = struct.unpack_from('<I', key, 0)[0] if len(key) >= 4 else -1
|
|
690
|
+
ok = kid == sid
|
|
691
|
+
else:
|
|
692
|
+
kh, kid = (struct.unpack_from('<II', key, 0)
|
|
693
|
+
if len(key) >= 8 else (-1, -1))
|
|
694
|
+
ok = kid == sid and kh == h
|
|
695
|
+
if not ok:
|
|
696
|
+
info['index_problems'].append(f'{label}: key does not match its '
|
|
697
|
+
f'entry data (id {sid})')
|
|
698
|
+
continue
|
|
699
|
+
seen.add(sid)
|
|
700
|
+
if entries.get(sid) != (h, off, length):
|
|
701
|
+
info['index_problems'].append(f'{label}: entry for id {sid} '
|
|
702
|
+
'disagrees with $SDS')
|
|
703
|
+
for sid in entries.keys() - seen:
|
|
704
|
+
info['index_problems'].append(f'{label}: id {sid} missing from index')
|
|
705
|
+
|
|
706
|
+
# every in-use file's security_id must exist in $SDS (id 0 = none) —
|
|
707
|
+
# the referenced-id map comes from the shared $MFT survey
|
|
708
|
+
s = survey if survey and survey.get('sec_refs') is not None \
|
|
709
|
+
else self.mft_survey(want_security=True)
|
|
710
|
+
info['files_checked'] = sum(len(r) for r in s['sec_refs'].values())
|
|
711
|
+
for sid, records in sorted(s['sec_refs'].items()):
|
|
712
|
+
if sid not in entries:
|
|
713
|
+
info['ref_missing'].append(
|
|
714
|
+
f'security id {sid} referenced by {len(records)} file(s) '
|
|
715
|
+
f'(e.g. record {records[0]}) but absent from $SDS')
|
|
716
|
+
return info
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
# -- $Extend view indexes: $Reparse ($R) and $ObjId ($O) vs the records --
|
|
720
|
+
|
|
721
|
+
def view_index_check(self, survey: dict | None = None) -> dict:
|
|
722
|
+
'''Walk the $Reparse and $ObjId view indexes (key-ordered) and check
|
|
723
|
+
both directions: every entry points at a live record carrying the
|
|
724
|
+
matching attribute, and every record carrying it has an entry.'''
|
|
725
|
+
out = {'problems': [], 'notes': [], 'checked': 0}
|
|
726
|
+
s = survey or self.mft_survey()
|
|
727
|
+
for path, name_const, attr_want, kind, recs in (
|
|
728
|
+
('/$Extend/$Reparse', R_NAME, 0xC0, '$Reparse', s['reparse_recs']),
|
|
729
|
+
('/$Extend/$ObjId', O_NAME, 0x40, '$ObjId', s['objid_recs'])):
|
|
730
|
+
try:
|
|
731
|
+
no = self.resolve(path)
|
|
732
|
+
root = self._read_whole_attr(no, AT_INDEX_ROOT, name_const, 2)
|
|
733
|
+
except NtfsError:
|
|
734
|
+
out['notes'].append(f'{kind}: absent')
|
|
735
|
+
if recs:
|
|
736
|
+
out['problems'].append(f'{kind}: index absent but {len(recs)} '
|
|
737
|
+
'record(s) carry the attribute')
|
|
738
|
+
continue
|
|
739
|
+
alloc = bmp = None
|
|
740
|
+
try:
|
|
741
|
+
alloc = self._read_whole_attr(no, AT_INDEX_ALLOCATION, name_const, 2)
|
|
742
|
+
bmp = self._read_whole_attr(no, AT_BITMAP, name_const, 2)
|
|
743
|
+
except NtfsError:
|
|
744
|
+
pass
|
|
745
|
+
key_fn = (lambda k: struct.unpack_from(f'<{len(k) // 4}I', k)
|
|
746
|
+
if len(k) >= 4 else ())
|
|
747
|
+
entries, probs = _view_index_entries(root, alloc, bmp, kind, key_fn=key_fn)
|
|
748
|
+
out['problems'] += probs
|
|
749
|
+
seen: set[int] = set()
|
|
750
|
+
for key, data in entries:
|
|
751
|
+
out['checked'] += 1
|
|
752
|
+
if kind == '$Reparse' and len(key) >= 12:
|
|
753
|
+
mref = struct.unpack_from('<Q', key, 4)[0]
|
|
754
|
+
elif kind == '$ObjId' and len(data) >= 8:
|
|
755
|
+
mref = struct.unpack_from('<Q', data, 0)[0]
|
|
756
|
+
else:
|
|
757
|
+
out['problems'].append(f'{kind}: malformed entry')
|
|
758
|
+
continue
|
|
759
|
+
m_no = mref & MREF_MASK
|
|
760
|
+
r = self._load_record(m_no)
|
|
761
|
+
ok = (r is not None
|
|
762
|
+
and struct.unpack_from('<H', r, 22)[0] & 1
|
|
763
|
+
and struct.unpack_from('<H', r, 16)[0] == mref >> 48
|
|
764
|
+
and _has_attr(r, attr_want))
|
|
765
|
+
if ok:
|
|
766
|
+
seen.add(m_no)
|
|
767
|
+
else:
|
|
768
|
+
out['problems'].append(f'{kind}: entry points at record {m_no} '
|
|
769
|
+
'which is dead, reused or lacks the attribute')
|
|
770
|
+
for m_no in sorted(recs - seen)[:8]:
|
|
771
|
+
out['problems'].append(f'{kind}: record {m_no} carries the attribute '
|
|
772
|
+
'but has no index entry')
|
|
773
|
+
return out
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
# -- lost files: in-use records no index entry references (chkdsk's
|
|
777
|
+
# "recovering orphaned file") --
|
|
778
|
+
|
|
779
|
+
def find_lost_files(self, referenced: set[int]) -> list[dict]:
|
|
780
|
+
'''Every in-use base record (past the system range) carrying $FILE_NAME
|
|
781
|
+
attributes but absent from `referenced` — the set of records that
|
|
782
|
+
some valid index entry points at. parent_ok means the claimed parent
|
|
783
|
+
is a live directory with a matching sequence, i.e. reconnectable.'''
|
|
784
|
+
lost: list[dict] = []
|
|
785
|
+
rec_size = self.mft_record_size()
|
|
786
|
+
chunk_size, offset = 1 << 20, 0
|
|
787
|
+
while True:
|
|
788
|
+
chunk = self._mft_bulk(offset, chunk_size)
|
|
789
|
+
usable = len(chunk) - (len(chunk) % rec_size)
|
|
790
|
+
if not usable:
|
|
791
|
+
break
|
|
792
|
+
chunk = chunk[:usable]
|
|
793
|
+
for rec_off in range(0, usable, rec_size):
|
|
794
|
+
rec_no = (offset + rec_off) // rec_size
|
|
795
|
+
if rec_no < 24 or rec_no in referenced:
|
|
796
|
+
continue # system range is never index-reconnected
|
|
797
|
+
raw = chunk[rec_off:rec_off + rec_size]
|
|
798
|
+
if raw[:4] != b'FILE':
|
|
799
|
+
continue
|
|
800
|
+
rec = bytearray(raw)
|
|
801
|
+
if not _apply_fixups(rec):
|
|
802
|
+
continue
|
|
803
|
+
flags = struct.unpack_from('<H', rec, 22)[0]
|
|
804
|
+
base = struct.unpack_from('<Q', rec, 32)[0]
|
|
805
|
+
if not flags & 1 or base & MREF_MASK:
|
|
806
|
+
continue
|
|
807
|
+
frozen = bytes(rec)
|
|
808
|
+
names = list(_file_name_attrs(frozen))
|
|
809
|
+
if not names:
|
|
810
|
+
continue
|
|
811
|
+
seq = struct.unpack_from('<H', frozen, 16)[0]
|
|
812
|
+
parent = struct.unpack_from('<Q', names[0], 0)[0]
|
|
813
|
+
p_no, p_seq = parent & MREF_MASK, parent >> 48
|
|
814
|
+
# reconnectable = the claimed parent is a live DIRECTORY whose
|
|
815
|
+
# sequence still matches (stricter than _mref_live)
|
|
816
|
+
p_rec = self._load_record(p_no)
|
|
817
|
+
parent_ok = False
|
|
818
|
+
if p_rec is not None:
|
|
819
|
+
p_flags = struct.unpack_from('<H', p_rec, 22)[0]
|
|
820
|
+
parent_ok = bool(p_flags & 1 and p_flags & 2
|
|
821
|
+
and struct.unpack_from('<H', p_rec, 16)[0] == p_seq)
|
|
822
|
+
lost.append({
|
|
823
|
+
'record': rec_no, 'seq': seq, 'parent': p_no,
|
|
824
|
+
'parent_ok': parent_ok, 'fn_list': [bytes(fn) for fn in names],
|
|
825
|
+
'name': names[0][66:66 + 2 * names[0][64]].decode(
|
|
826
|
+
'utf-16-le', 'replace')})
|
|
827
|
+
offset += usable
|
|
828
|
+
return lost
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
# -- chkdsk stage 4 support: attribute bad clusters to their owners --
|
|
832
|
+
|
|
833
|
+
def surface_owners(self, bad: set[int]) -> list[dict]:
|
|
834
|
+
'''Attribute unreadable clusters to their owning records by decoding
|
|
835
|
+
every in-use record's runlists raw (one bulk $MFT pass); whatever
|
|
836
|
+
no record claims is reported as free space.'''
|
|
837
|
+
order = sorted(bad)
|
|
838
|
+
owners: list[dict] = []
|
|
839
|
+
claimed: set[int] = set()
|
|
840
|
+
nc = self.nr_clusters()
|
|
841
|
+
rec_size = self.mft_record_size()
|
|
842
|
+
chunk_size, offset = 1 << 20, 0
|
|
843
|
+
while True:
|
|
844
|
+
chunk = self._mft_bulk(offset, chunk_size)
|
|
845
|
+
usable = len(chunk) - (len(chunk) % rec_size)
|
|
846
|
+
if not usable:
|
|
847
|
+
break
|
|
848
|
+
chunk = chunk[:usable]
|
|
849
|
+
for rec_off in range(0, usable, rec_size):
|
|
850
|
+
raw = chunk[rec_off:rec_off + rec_size]
|
|
851
|
+
if raw[:4] != b'FILE':
|
|
852
|
+
continue
|
|
853
|
+
rec = bytearray(raw)
|
|
854
|
+
if not _apply_fixups(rec):
|
|
855
|
+
continue
|
|
856
|
+
if not struct.unpack_from('<H', rec, 22)[0] & 1:
|
|
857
|
+
continue
|
|
858
|
+
frozen = bytes(rec)
|
|
859
|
+
rec_no = (offset + rec_off) // rec_size
|
|
860
|
+
a_off = struct.unpack_from('<H', frozen, 20)[0]
|
|
861
|
+
while a_off + 24 <= len(frozen):
|
|
862
|
+
attr_type = struct.unpack_from('<I', frozen, a_off)[0]
|
|
863
|
+
if attr_type == 0xFFFFFFFF:
|
|
864
|
+
break
|
|
865
|
+
length = struct.unpack_from('<I', frozen, a_off + 4)[0]
|
|
866
|
+
if length < 24 or a_off + length > len(frozen):
|
|
867
|
+
break
|
|
868
|
+
if frozen[a_off + 8] == 1:
|
|
869
|
+
_p, _ph, runs = _check_mapping_pairs(frozen, a_off, length, nc)
|
|
870
|
+
for lcn, run_len, _vcn in runs:
|
|
871
|
+
lo = bisect.bisect_left(order, lcn)
|
|
872
|
+
hi = bisect.bisect_right(order, lcn + run_len - 1)
|
|
873
|
+
if hi > lo:
|
|
874
|
+
hits = order[lo:hi]
|
|
875
|
+
names = ', '.join(
|
|
876
|
+
repr(fn[66:66 + 2 * fn[64]].decode('utf-16-le',
|
|
877
|
+
'replace'))
|
|
878
|
+
for fn in _file_name_attrs(frozen)) or '(no name)'
|
|
879
|
+
owners.append({'record': rec_no, 'attr': attr_type,
|
|
880
|
+
'names': names, 'clusters': hits})
|
|
881
|
+
claimed.update(hits)
|
|
882
|
+
a_off += length
|
|
883
|
+
offset += usable
|
|
884
|
+
free = sorted(bad - claimed)
|
|
885
|
+
if free:
|
|
886
|
+
owners.append({'record': None, 'attr': None,
|
|
887
|
+
'names': '(free space)', 'clusters': free})
|
|
888
|
+
return owners
|
|
889
|
+
|
|
890
|
+
|
|
891
|
+
def __enter__(self):
|
|
892
|
+
return self
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
def __exit__(self, *_exc):
|
|
896
|
+
self.close()
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
# -- NTFS name equality: this volume's $UpCase table, not Python casing --
|
|
900
|
+
|
|
901
|
+
def _upcase_table(self) -> array.array | None:
|
|
902
|
+
if self._upcase is None:
|
|
903
|
+
self._upcase = False # sticky: don't retry a failed load
|
|
904
|
+
try:
|
|
905
|
+
raw = self._read_whole_attr(FILE_UPCASE, AT_DATA)
|
|
906
|
+
except NtfsError:
|
|
907
|
+
raw = b''
|
|
908
|
+
if len(raw) >= 512:
|
|
909
|
+
table = array.array('H', raw[:len(raw) & ~1])
|
|
910
|
+
if sys.byteorder == 'big':
|
|
911
|
+
table.byteswap()
|
|
912
|
+
self._upcase = table
|
|
913
|
+
return self._upcase or None
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
def names_equal(self, a: str, b: str) -> bool:
|
|
917
|
+
'''Case-insensitive equality the way THIS volume defines it ($UpCase);
|
|
918
|
+
degrades to Python casing only if $UpCase cannot be read.'''
|
|
919
|
+
table = self._upcase_table()
|
|
920
|
+
if table is None:
|
|
921
|
+
return a.lower() == b.lower()
|
|
922
|
+
ea, eb = a.encode('utf-16-le'), b.encode('utf-16-le')
|
|
923
|
+
if len(ea) != len(eb):
|
|
924
|
+
return False
|
|
925
|
+
n = len(table)
|
|
926
|
+
return all((table[x] if x < n else x) == (table[y] if y < n else y)
|
|
927
|
+
for x, y in zip(struct.unpack(f'<{len(ea) // 2}H', ea),
|
|
928
|
+
struct.unpack(f'<{len(eb) // 2}H', eb)))
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
# -- the unified $MFT sweep: one bulk pass serves stages 1, 3 and 5 --
|
|
932
|
+
def mft_survey(self, want_used: bool = False, want_security: bool = False) -> dict:
|
|
933
|
+
'''One pass over $MFT computing stage-1 health (record counts, torn
|
|
934
|
+
records, attribute-chain and runlist problems — entries with a
|
|
935
|
+
'fix_mp_off' are safely auto-repairable torn truncates), and
|
|
936
|
+
optionally the stage-5 used-cluster map from the decoded runlists
|
|
937
|
+
and the stage-3 map of referenced security ids. Reading $MFT once
|
|
938
|
+
instead of once per stage (and decoding runlists raw instead of
|
|
939
|
+
729k per-inode round-trips) is the main /f speedup.'''
|
|
940
|
+
nc = self.nr_clusters()
|
|
941
|
+
rec_size = self.mft_record_size()
|
|
942
|
+
total = in_use = dirs = torn = 0
|
|
943
|
+
problems: list[dict] = []
|
|
944
|
+
used = bytearray((nc + 7) // 8) if want_used else None
|
|
945
|
+
# per-RECORD in-use bits (for the $MFT $BITMAP reconciliation) — not
|
|
946
|
+
# to be confused with `used`, which is the per-CLUSTER map for stage 5
|
|
947
|
+
n_slots = self._mft_size // rec_size
|
|
948
|
+
rec_used = bytearray((n_slots + 7) // 8) if want_used else None
|
|
949
|
+
map_failures: list[str] = []
|
|
950
|
+
sec_refs: dict[int, list[int]] = {}
|
|
951
|
+
reparse_recs: set[int] = set()
|
|
952
|
+
objid_recs: set[int] = set()
|
|
953
|
+
chunk_size, offset = 1 << 20, 0
|
|
954
|
+
while True:
|
|
955
|
+
chunk = self._mft_bulk(offset, chunk_size)
|
|
956
|
+
usable = len(chunk) - (len(chunk) % rec_size)
|
|
957
|
+
if not usable:
|
|
958
|
+
break
|
|
959
|
+
chunk = chunk[:usable]
|
|
960
|
+
for rec_off in range(0, usable, rec_size):
|
|
961
|
+
raw = chunk[rec_off:rec_off + rec_size]
|
|
962
|
+
if raw[:4] != b'FILE':
|
|
963
|
+
continue
|
|
964
|
+
total += 1
|
|
965
|
+
rec_no = (offset + rec_off) // rec_size
|
|
966
|
+
rec = bytearray(raw)
|
|
967
|
+
if not _apply_fixups(rec):
|
|
968
|
+
torn += 1
|
|
969
|
+
# flags live in the first sector — readable even when torn
|
|
970
|
+
if struct.unpack_from('<H', raw, 22)[0] & 1 and want_used:
|
|
971
|
+
map_failures.append(f'record {rec_no}: torn but in use — '
|
|
972
|
+
'its clusters are unknowable')
|
|
973
|
+
# keep its bitmap bit set: clearing a slot that still
|
|
974
|
+
# claims in-use would invite reuse under a live record
|
|
975
|
+
rec_used[rec_no >> 3] |= 1 << (rec_no & 7)
|
|
976
|
+
continue
|
|
977
|
+
flags = struct.unpack_from('<H', rec, 22)[0]
|
|
978
|
+
in_use += flags & 1
|
|
979
|
+
dirs += bool(flags & 1) and bool(flags & 2)
|
|
980
|
+
if flags & 1 and want_used:
|
|
981
|
+
rec_used[rec_no >> 3] |= 1 << (rec_no & 7)
|
|
982
|
+
if not flags & 1:
|
|
983
|
+
continue
|
|
984
|
+
frozen = bytes(rec)
|
|
985
|
+
base = struct.unpack_from('<Q', frozen, 32)[0]
|
|
986
|
+
if base & MREF_MASK and not self._mref_live(base):
|
|
987
|
+
# extension record whose base is dead or reused
|
|
988
|
+
problems.append({'record': rec_no, 'attr_type': 0,
|
|
989
|
+
'fix_mp_off': None, 'frozen': frozen,
|
|
990
|
+
'problem': f'orphaned extension record: base '
|
|
991
|
+
f'{base & MREF_MASK} is dead or reused'})
|
|
992
|
+
fn_count, has_attr_list = 0, False
|
|
993
|
+
a_off = struct.unpack_from('<H', frozen, 20)[0]
|
|
994
|
+
while a_off + 8 <= len(frozen):
|
|
995
|
+
attr_type = struct.unpack_from('<I', frozen, a_off)[0]
|
|
996
|
+
if attr_type == 0xFFFFFFFF:
|
|
997
|
+
break
|
|
998
|
+
length = struct.unpack_from('<I', frozen, a_off + 4)[0]
|
|
999
|
+
if length < 24 or a_off + length > len(frozen):
|
|
1000
|
+
problems.append({'record': rec_no, 'attr_type': attr_type,
|
|
1001
|
+
'fix_mp_off': None, 'frozen': frozen,
|
|
1002
|
+
'problem': f'broken attribute chain at +{a_off}'})
|
|
1003
|
+
if want_used:
|
|
1004
|
+
map_failures.append(f'record {rec_no}: broken attribute chain')
|
|
1005
|
+
break
|
|
1006
|
+
if attr_type == AT_FILE_NAME and frozen[a_off + 8] == 0:
|
|
1007
|
+
fn_count += 1
|
|
1008
|
+
elif attr_type == 0xC0:
|
|
1009
|
+
reparse_recs.add(rec_no)
|
|
1010
|
+
elif attr_type == 0x40:
|
|
1011
|
+
objid_recs.add(rec_no)
|
|
1012
|
+
elif attr_type == AT_ATTRIBUTE_LIST and frozen[a_off + 8] == 0:
|
|
1013
|
+
has_attr_list = True
|
|
1014
|
+
v_len = struct.unpack_from('<I', frozen, a_off + 16)[0]
|
|
1015
|
+
v_ofs = struct.unpack_from('<H', frozen, a_off + 20)[0]
|
|
1016
|
+
listing = frozen[a_off + v_ofs:a_off + v_ofs + v_len]
|
|
1017
|
+
pos = 0
|
|
1018
|
+
while pos + 26 <= len(listing):
|
|
1019
|
+
e_len = struct.unpack_from('<H', listing, pos + 4)[0]
|
|
1020
|
+
if e_len < 26:
|
|
1021
|
+
problems.append({'record': rec_no, 'attr_type': 0x20,
|
|
1022
|
+
'fix_mp_off': None, 'frozen': frozen,
|
|
1023
|
+
'problem': 'broken $ATTRIBUTE_LIST entry'})
|
|
1024
|
+
break
|
|
1025
|
+
lref = struct.unpack_from('<Q', listing, pos + 16)[0]
|
|
1026
|
+
l_no = lref & MREF_MASK
|
|
1027
|
+
if l_no != rec_no and not self._mref_live(lref):
|
|
1028
|
+
problems.append(
|
|
1029
|
+
{'record': rec_no, 'attr_type': 0x20,
|
|
1030
|
+
'fix_mp_off': None, 'frozen': frozen,
|
|
1031
|
+
'problem': '$ATTRIBUTE_LIST points at dead/'
|
|
1032
|
+
f'reused extension record {l_no}'})
|
|
1033
|
+
pos += e_len
|
|
1034
|
+
elif attr_type == AT_ATTRIBUTE_LIST:
|
|
1035
|
+
has_attr_list = True # non-resident listing: skip content
|
|
1036
|
+
if (want_security and attr_type == AT_STANDARD_INFORMATION
|
|
1037
|
+
and frozen[a_off + 8] == 0):
|
|
1038
|
+
value_len = struct.unpack_from('<I', frozen, a_off + 16)[0]
|
|
1039
|
+
value_ofs = struct.unpack_from('<H', frozen, a_off + 20)[0]
|
|
1040
|
+
if value_len >= 72:
|
|
1041
|
+
sid = struct.unpack_from('<I', frozen,
|
|
1042
|
+
a_off + value_ofs + 52)[0]
|
|
1043
|
+
if sid:
|
|
1044
|
+
sec_refs.setdefault(sid, []).append(rec_no)
|
|
1045
|
+
if frozen[a_off + 8] == 1: # non-resident
|
|
1046
|
+
problem, phantom, runs = _check_mapping_pairs(
|
|
1047
|
+
frozen, a_off, length, nc)
|
|
1048
|
+
if problem:
|
|
1049
|
+
fix = None
|
|
1050
|
+
if phantom:
|
|
1051
|
+
sizes = struct.unpack_from('<QQQ', frozen, a_off + 40)
|
|
1052
|
+
mp_abs = a_off + struct.unpack_from(
|
|
1053
|
+
'<H', frozen, a_off + 32)[0]
|
|
1054
|
+
if sizes == (0, 0, 0) and mp_abs % 512 < 510:
|
|
1055
|
+
fix = mp_abs
|
|
1056
|
+
problems.append({'record': rec_no, 'attr_type': attr_type,
|
|
1057
|
+
'problem': problem, 'fix_mp_off': fix,
|
|
1058
|
+
'frozen': frozen})
|
|
1059
|
+
# phantom runs are never live data (header says empty)
|
|
1060
|
+
# so they don't make the used map incomplete
|
|
1061
|
+
if want_used and not phantom:
|
|
1062
|
+
map_failures.append(
|
|
1063
|
+
f'record {rec_no} attr {attr_type:#x}: {problem}')
|
|
1064
|
+
elif want_used:
|
|
1065
|
+
for lcn, run_len, _vcn in runs:
|
|
1066
|
+
_set_run(used, lcn, run_len)
|
|
1067
|
+
a_off += length
|
|
1068
|
+
# hard-link count vs the record's own $FILE_NAME attrs (spilled
|
|
1069
|
+
# attr lists skipped: names may live in extension records)
|
|
1070
|
+
if not has_attr_list and fn_count and not base & MREF_MASK:
|
|
1071
|
+
link_count = struct.unpack_from('<H', frozen, 18)[0]
|
|
1072
|
+
if link_count != fn_count:
|
|
1073
|
+
problems.append({'record': rec_no, 'attr_type': AT_FILE_NAME,
|
|
1074
|
+
'fix_mp_off': None, 'frozen': frozen,
|
|
1075
|
+
'problem': f'hard-link count {link_count} but '
|
|
1076
|
+
f'{fn_count} $FILE_NAME attribute(s)'})
|
|
1077
|
+
offset += usable
|
|
1078
|
+
for prob in problems:
|
|
1079
|
+
prob['names'] = ', '.join(
|
|
1080
|
+
repr(fn[66:66 + 2 * fn[64]].decode('utf-16-le', 'replace'))
|
|
1081
|
+
for fn in _file_name_attrs(prob.pop('frozen'))) or '(no name)'
|
|
1082
|
+
return {'total': total, 'in_use': in_use, 'dirs': dirs, 'torn': torn,
|
|
1083
|
+
'runlist_problems': problems, 'used': used,
|
|
1084
|
+
'rec_used': rec_used, 'n_slots': n_slots,
|
|
1085
|
+
'map_failures': map_failures, 'sec_refs': sec_refs,
|
|
1086
|
+
'reparse_recs': reparse_recs, 'objid_recs': objid_recs}
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
def _entry_status(self, dir_mft_no: int, entry: dict) -> str:
|
|
1090
|
+
'''Validate one index entry against the raw MFT record — chkdsk stage-2
|
|
1091
|
+
semantics. Deliberately NOT ntfs_inode_open(mref): a cached inode hit
|
|
1092
|
+
skips the sequence check, so a stale entry can scan "ok" whenever the
|
|
1093
|
+
record's current owner was opened earlier on this volume handle.'''
|
|
1094
|
+
|
|
1095
|
+
raw = self.read_record(entry['record'])
|
|
1096
|
+
if raw is None or raw[:4] != b'FILE':
|
|
1097
|
+
return 'DANGLING (record unreadable)'
|
|
1098
|
+
rec = bytearray(raw)
|
|
1099
|
+
if not _apply_fixups(rec):
|
|
1100
|
+
return 'DANGLING (torn record)'
|
|
1101
|
+
flags = struct.unpack_from('<H', rec, 22)[0]
|
|
1102
|
+
entry['is_dir'] = bool(flags & 2)
|
|
1103
|
+
if not flags & 1:
|
|
1104
|
+
return 'DANGLING (record not in use)'
|
|
1105
|
+
rec_seq = struct.unpack_from('<H', rec, 16)[0]
|
|
1106
|
+
if rec_seq != entry['seq']:
|
|
1107
|
+
return f'DANGLING (seq mismatch: dirent {entry["seq"]} vs record {rec_seq})'
|
|
1108
|
+
frozen = bytes(rec)
|
|
1109
|
+
for fn in _file_name_attrs(frozen):
|
|
1110
|
+
parent = struct.unpack_from('<Q', fn, 0)[0]
|
|
1111
|
+
fn_name = fn[66:66 + 2 * fn[64]].decode('utf-16-le', 'replace')
|
|
1112
|
+
if parent & MREF_MASK == dir_mft_no and self.names_equal(fn_name, entry['name']):
|
|
1113
|
+
return 'ok'
|
|
1114
|
+
# $FILE_NAME may live in an extension record when an $ATTRIBUTE_LIST
|
|
1115
|
+
# exists — fall back to the full attr-list enumeration for those.
|
|
1116
|
+
if _has_attr(frozen, AT_ATTRIBUTE_LIST):
|
|
1117
|
+
info = self.record_info(entry['record'])
|
|
1118
|
+
if any(self.names_equal(n['name'], entry['name'])
|
|
1119
|
+
and n['parent_record'] == dir_mft_no
|
|
1120
|
+
for n in info.get('names', [])):
|
|
1121
|
+
return 'ok'
|
|
1122
|
+
return 'DANGLING (cross-linked: record does not carry this name)'
|
|
1123
|
+
|
|
1124
|
+
|
|
1125
|
+
def _classify_found(self, dir_mft_no: int, name: str, found: dict) -> dict:
|
|
1126
|
+
result = {'name': name, 'dir_record': dir_mft_no,
|
|
1127
|
+
'dirent_record': found['record'], 'dirent_seq': found['seq']}
|
|
1128
|
+
|
|
1129
|
+
# Healthy = the record header matches the dirent's (record, seq) AND the
|
|
1130
|
+
# record carries this (name, parent) in a $FILE_NAME — chkdsk validates
|
|
1131
|
+
# both. Deliberately NOT ntfs_inode_open(mref): a prior seq-bypass open
|
|
1132
|
+
# (ours, below) leaves the inode cached and a cached hit skips the seq
|
|
1133
|
+
# check, so that test would go stale after the first classify.
|
|
1134
|
+
record = self.record_info(found['record'])
|
|
1135
|
+
result['record_info'] = record
|
|
1136
|
+
names_match = record.get('open') and any(
|
|
1137
|
+
self.names_equal(entry['name'], name)
|
|
1138
|
+
and entry['parent_record'] == dir_mft_no for entry in record['names'])
|
|
1139
|
+
if record['open'] and record.get('in_use') and record['seq'] == found['seq']:
|
|
1140
|
+
if names_match:
|
|
1141
|
+
result.update(state='healthy', action='nothing to do')
|
|
1142
|
+
else:
|
|
1143
|
+
result.update(
|
|
1144
|
+
state='cross-linked',
|
|
1145
|
+
action='rm ONLY — the dirent points into a live record that does '
|
|
1146
|
+
'not carry this name; the real file lives elsewhere')
|
|
1147
|
+
elif not record['open'] or not record.get('in_use'):
|
|
1148
|
+
result.update(
|
|
1149
|
+
state='record-free',
|
|
1150
|
+
action='rm — the entry is the only leftover; nothing else to clean')
|
|
1151
|
+
elif names_match:
|
|
1152
|
+
result.update(
|
|
1153
|
+
state='orphan',
|
|
1154
|
+
action='purge — record still belongs to this name; one call frees '
|
|
1155
|
+
'dirent, record and clusters')
|
|
1156
|
+
else:
|
|
1157
|
+
names = ', '.join(repr(entry['name']) for entry in record['names']) or '(none)'
|
|
1158
|
+
result.update(
|
|
1159
|
+
state='record-reused',
|
|
1160
|
+
action=f'rm ONLY — the record now belongs to a live file ({names}); '
|
|
1161
|
+
'do not touch it')
|
|
1162
|
+
return result
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
# -- chkdsk-style index rebuild: recover a directory from the MFT side --
|
|
1166
|
+
|
|
1167
|
+
def probe_dir_no(self, mft_no: int) -> dict:
|
|
1168
|
+
try:
|
|
1169
|
+
entries = [e for e in self.scan_dir_no(mft_no) if e['status'] != 'dir-self']
|
|
1170
|
+
return {'readable': True, 'entries': entries}
|
|
1171
|
+
except NtfsError as exc:
|
|
1172
|
+
return {'readable': False, 'error': str(exc)}
|
|
1173
|
+
|
|
1174
|
+
|
|
1175
|
+
def probe_dir(self, path: str) -> dict:
|
|
1176
|
+
'''Try to walk the index; a torn INDX block surfaces as a readdir error '''
|
|
1177
|
+
|
|
1178
|
+
try:
|
|
1179
|
+
return self.probe_dir_no(self._resolve(path))
|
|
1180
|
+
except NtfsError as exc:
|
|
1181
|
+
return {'readable': False, 'error': str(exc)}
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
def children_from_mft(self, dir_mft_no: int) -> dict:
|
|
1185
|
+
'''Every in-use base MFT record whose $FILE_NAME claims dir_mft_no as
|
|
1186
|
+
parent — the same redundancy chkdsk rebuilds a torn index from '''
|
|
1187
|
+
|
|
1188
|
+
children: list[dict] = []
|
|
1189
|
+
unreadable = 0
|
|
1190
|
+
dir_seq = None
|
|
1191
|
+
rec_size = self.mft_record_size()
|
|
1192
|
+
chunk_size, offset = 1 << 20, 0
|
|
1193
|
+
while True:
|
|
1194
|
+
chunk = self._mft_bulk(offset, chunk_size)
|
|
1195
|
+
# Only advance by whole records: a short read must not shift
|
|
1196
|
+
# every subsequent record off its alignment.
|
|
1197
|
+
usable = len(chunk) - (len(chunk) % rec_size)
|
|
1198
|
+
if not usable:
|
|
1199
|
+
break # tail shorter than one record
|
|
1200
|
+
chunk = chunk[:usable]
|
|
1201
|
+
for rec_off in range(0, usable, rec_size):
|
|
1202
|
+
raw = chunk[rec_off:rec_off + rec_size]
|
|
1203
|
+
if raw[:4] != b'FILE':
|
|
1204
|
+
continue
|
|
1205
|
+
rec_no = (offset + rec_off) // rec_size
|
|
1206
|
+
rec = bytearray(raw)
|
|
1207
|
+
if not _apply_fixups(rec):
|
|
1208
|
+
unreadable += 1
|
|
1209
|
+
continue
|
|
1210
|
+
flags = struct.unpack_from('<H', rec, 22)[0]
|
|
1211
|
+
base = struct.unpack_from('<Q', rec, 32)[0]
|
|
1212
|
+
seq = struct.unpack_from('<H', rec, 16)[0]
|
|
1213
|
+
if rec_no == dir_mft_no:
|
|
1214
|
+
dir_seq = seq
|
|
1215
|
+
if not flags & 1 or base & MREF_MASK: # free, or extension record
|
|
1216
|
+
continue
|
|
1217
|
+
for fn in _file_name_attrs(rec):
|
|
1218
|
+
parent = struct.unpack_from('<Q', fn, 0)[0]
|
|
1219
|
+
if parent & MREF_MASK != dir_mft_no or rec_no == dir_mft_no:
|
|
1220
|
+
continue
|
|
1221
|
+
n_len, n_type = fn[64], fn[65]
|
|
1222
|
+
children.append({
|
|
1223
|
+
'record': rec_no, 'seq': seq, 'type': n_type,
|
|
1224
|
+
'parent_seq': parent >> 48,
|
|
1225
|
+
'name': fn[66:66 + 2 * n_len].decode('utf-16-le', 'replace'),
|
|
1226
|
+
'fn_bytes': bytes(fn),
|
|
1227
|
+
})
|
|
1228
|
+
offset += usable
|
|
1229
|
+
if dir_seq is None:
|
|
1230
|
+
raise NtfsError(0, f'directory record {dir_mft_no} not found in the $MFT scan')
|
|
1231
|
+
# A name whose parent reference carries a stale sequence belongs to a
|
|
1232
|
+
# previous incarnation of this directory — chkdsk would not resurrect
|
|
1233
|
+
# it into the rebuilt index, and neither do we.
|
|
1234
|
+
live = [c for c in children if c['parent_seq'] == dir_seq]
|
|
1235
|
+
return {'children': live, 'dir_seq': dir_seq,
|
|
1236
|
+
'stale_parent': len(children) - len(live),
|
|
1237
|
+
'unreadable_records': unreadable}
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
def __init__(self, device: str, readonly: bool = True, base: int = 0):
|
|
1241
|
+
if not readonly:
|
|
1242
|
+
raise SystemExit('RawVolume is read-only — use RawVolumeRW for repairs')
|
|
1243
|
+
self.device = device
|
|
1244
|
+
self._upcase = None
|
|
1245
|
+
self._base = base # byte offset of the volume within the device
|
|
1246
|
+
self._fd = os.open(device, os.O_RDONLY | _O_BINARY)
|
|
1247
|
+
boot = _pread(self._fd, 512, self._base)
|
|
1248
|
+
if boot[3:7] != b'NTFS':
|
|
1249
|
+
raise NtfsError(0, f'{device}: no NTFS boot sector')
|
|
1250
|
+
bps = struct.unpack_from('<H', boot, 11)[0]
|
|
1251
|
+
spc = boot[13]
|
|
1252
|
+
spc = 2 ** (256 - spc) if spc > 0x80 else spc
|
|
1253
|
+
self._cluster_size = bps * spc
|
|
1254
|
+
self._nr_clusters = struct.unpack_from('<Q', boot, 40)[0] // spc
|
|
1255
|
+
mft_lcn = struct.unpack_from('<Q', boot, 48)[0]
|
|
1256
|
+
raw = struct.unpack_from('<b', boot, 64)[0]
|
|
1257
|
+
self._rec_size = 2 ** -raw if raw < 0 else raw * self._cluster_size
|
|
1258
|
+
if self._rec_size not in (1024, 2048, 4096):
|
|
1259
|
+
raise NtfsError(0, f'implausible MFT record size {self._rec_size}')
|
|
1260
|
+
# bootstrap caches + provisional $MFT map (just its first clusters,
|
|
1261
|
+
# enough to read record 0 and discover the real runlist)
|
|
1262
|
+
import collections
|
|
1263
|
+
self._blk_cache = collections.OrderedDict()
|
|
1264
|
+
self._attr_memo = {}
|
|
1265
|
+
self._mft_runs = [(mft_lcn, 4, 0)]
|
|
1266
|
+
self._mft_size = 4 * self._cluster_size
|
|
1267
|
+
self._mft_starts = [0]
|
|
1268
|
+
rec0 = self.read_record(0)
|
|
1269
|
+
if rec0 is None or rec0[:4] != b'FILE':
|
|
1270
|
+
raise NtfsError(0, 'cannot read $MFT record 0')
|
|
1271
|
+
self._mft_runs = self._stream_runs(0, AT_DATA, None)[1]
|
|
1272
|
+
self._mft_size = sum(r[1] for r in self._mft_runs) * self._cluster_size
|
|
1273
|
+
self._mft_starts = [r[2] * self._cluster_size for r in self._mft_runs]
|
|
1274
|
+
self._blk_cache.clear() # drop blocks read via the provisional map
|
|
1275
|
+
self._attr_memo.clear()
|
|
1276
|
+
try:
|
|
1277
|
+
vi = self._attr_value(3, 0x70, None)
|
|
1278
|
+
if vi and len(vi) >= 12 and struct.unpack_from('<H', vi, 10)[0] & 1:
|
|
1279
|
+
print(f'WARNING: {device} is marked dirty (unreplayed $LogFile) '
|
|
1280
|
+
'— read results may be slightly stale', file=sys.stderr)
|
|
1281
|
+
except NtfsError:
|
|
1282
|
+
pass
|
|
1283
|
+
|
|
1284
|
+
def close(self) -> None:
|
|
1285
|
+
if getattr(self, '_fd', None) is not None:
|
|
1286
|
+
os.close(self._fd)
|
|
1287
|
+
self._fd = None
|
|
1288
|
+
|
|
1289
|
+
# -- dirty flag: $VOLUME_INFORMATION (0x70) of $Volume (record 3) --
|
|
1290
|
+
|
|
1291
|
+
def _volume_info(self):
|
|
1292
|
+
rec = self._load_record(3)
|
|
1293
|
+
if rec is None:
|
|
1294
|
+
raise NtfsError(errno.EIO, '$Volume record unreadable')
|
|
1295
|
+
for a, t, _ln in _attrs(rec):
|
|
1296
|
+
if t == 0x70 and rec[a + 8] == 0:
|
|
1297
|
+
v_ofs = struct.unpack_from('<H', rec, a + 20)[0]
|
|
1298
|
+
return rec, a + v_ofs + 10 # u16 flags inside the value
|
|
1299
|
+
raise NtfsError(errno.ENOENT, '$VOLUME_INFORMATION not found')
|
|
1300
|
+
|
|
1301
|
+
def _dirty_flag(self) -> bool:
|
|
1302
|
+
rec, off = self._volume_info()
|
|
1303
|
+
return bool(struct.unpack_from('<H', rec, off)[0] & 1)
|
|
1304
|
+
|
|
1305
|
+
def volume_label(self) -> str:
|
|
1306
|
+
'''$VOLUME_NAME (0x60) of $Volume — UTF-16LE, may be absent (no label).'''
|
|
1307
|
+
rec = self._load_record(3)
|
|
1308
|
+
if rec is None:
|
|
1309
|
+
return ''
|
|
1310
|
+
for a, t, _ln in _attrs(rec):
|
|
1311
|
+
if t == 0x60 and rec[a + 8] == 0:
|
|
1312
|
+
vlen = struct.unpack_from('<I', rec, a + 16)[0]
|
|
1313
|
+
vofs = struct.unpack_from('<H', rec, a + 20)[0]
|
|
1314
|
+
return bytes(rec[a + vofs:a + vofs + vlen]).decode('utf-16-le',
|
|
1315
|
+
'replace')
|
|
1316
|
+
return ''
|
|
1317
|
+
|
|
1318
|
+
# -- primitives --
|
|
1319
|
+
|
|
1320
|
+
def _dev_read(self, offset: int, size: int) -> bytes:
|
|
1321
|
+
return _pread(self._fd, size, self._base + offset)
|
|
1322
|
+
|
|
1323
|
+
def _runs_read(self, runs, offset: int, size: int) -> bytes:
|
|
1324
|
+
'''Read [offset, offset+size) of a stream laid out by vcn-ordered runs;
|
|
1325
|
+
holes (vcns no run covers) read as zeros. One pread per overlapping
|
|
1326
|
+
run, bisected start, zero-copy when one run covers the whole range.'''
|
|
1327
|
+
csz = self._cluster_size
|
|
1328
|
+
end = offset + size
|
|
1329
|
+
first = 0
|
|
1330
|
+
if len(runs) > 8: # skip straight to the first candidate run
|
|
1331
|
+
starts = (self._mft_starts if runs is self._mft_runs
|
|
1332
|
+
else [r[2] * csz for r in runs])
|
|
1333
|
+
first = max(bisect.bisect_right(starts, offset) - 1, 0)
|
|
1334
|
+
out = None
|
|
1335
|
+
for lcn, run_len, vcn in runs[first:]:
|
|
1336
|
+
r_start = vcn * csz
|
|
1337
|
+
if r_start >= end:
|
|
1338
|
+
break # runs are vcn-sorted: nothing further can overlap
|
|
1339
|
+
r_end = r_start + run_len * csz
|
|
1340
|
+
lo, hi = max(offset, r_start), min(end, r_end)
|
|
1341
|
+
if lo >= hi:
|
|
1342
|
+
continue
|
|
1343
|
+
data = _pread(self._fd, hi - lo, self._base + lcn * csz + (lo - r_start))
|
|
1344
|
+
if lo == offset and hi == end:
|
|
1345
|
+
return data # common case: whole range inside one run
|
|
1346
|
+
if out is None:
|
|
1347
|
+
out = bytearray(size)
|
|
1348
|
+
out[lo - offset:lo - offset + len(data)] = data
|
|
1349
|
+
return bytes(out) if out is not None else b'\x00' * size
|
|
1350
|
+
|
|
1351
|
+
def mft_record_size(self) -> int:
|
|
1352
|
+
return self._rec_size
|
|
1353
|
+
|
|
1354
|
+
BLOCK = 1 << 16 # record-read granularity: 64 KiB, LRU-cached
|
|
1355
|
+
|
|
1356
|
+
def _mft_bulk(self, offset: int, size: int) -> bytes:
|
|
1357
|
+
if offset >= self._mft_size:
|
|
1358
|
+
return b''
|
|
1359
|
+
return self._runs_read(self._mft_runs, offset,
|
|
1360
|
+
min(size, self._mft_size - offset))
|
|
1361
|
+
|
|
1362
|
+
def read_record(self, rec_no: int) -> bytes | None:
|
|
1363
|
+
'''64 KiB block-cached record reads. Unlike a naive chunk cache
|
|
1364
|
+
(whose misses re-read everything), a miss here
|
|
1365
|
+
is one pread served by the kernel page cache — amplification is a
|
|
1366
|
+
memcpy, and the OrderedDict LRU evicts one block, not everything.'''
|
|
1367
|
+
off = rec_no * self._rec_size
|
|
1368
|
+
bno, boff = divmod(off, self.BLOCK)
|
|
1369
|
+
cache = self._blk_cache
|
|
1370
|
+
blk = cache.get(bno)
|
|
1371
|
+
if blk is None:
|
|
1372
|
+
want = min(self.BLOCK, self._mft_size - bno * self.BLOCK)
|
|
1373
|
+
if want <= 0:
|
|
1374
|
+
return None
|
|
1375
|
+
if len(cache) >= 512: # 32 MiB cap
|
|
1376
|
+
cache.popitem(last=False)
|
|
1377
|
+
blk = cache[bno] = self._mft_bulk(bno * self.BLOCK, want)
|
|
1378
|
+
else:
|
|
1379
|
+
cache.move_to_end(bno)
|
|
1380
|
+
rec = blk[boff:boff + self._rec_size]
|
|
1381
|
+
return rec if len(rec) == self._rec_size else None
|
|
1382
|
+
|
|
1383
|
+
# -- raw attribute access (follows $ATTRIBUTE_LIST) --
|
|
1384
|
+
|
|
1385
|
+
def _record_attrs(self, rec_no: int) -> list:
|
|
1386
|
+
memo = self._attr_memo
|
|
1387
|
+
got = memo.get(rec_no)
|
|
1388
|
+
if got is None:
|
|
1389
|
+
if len(memo) >= 2048:
|
|
1390
|
+
memo.clear()
|
|
1391
|
+
got = memo[rec_no] = list(self._record_attrs_uncached(rec_no))
|
|
1392
|
+
return got
|
|
1393
|
+
|
|
1394
|
+
def _record_attrs_uncached(self, rec_no: int):
|
|
1395
|
+
'''Yield (frozen_record, attr_off, attr_len) for every attribute of
|
|
1396
|
+
this inode, base record first, then extension records in $ATTRIBUTE_
|
|
1397
|
+
LIST order (each extension yields its own attributes).'''
|
|
1398
|
+
rec = self._load_record(rec_no)
|
|
1399
|
+
if rec is None:
|
|
1400
|
+
raise NtfsError(errno.EIO, f'record {rec_no} unreadable or torn')
|
|
1401
|
+
frozen = bytes(rec)
|
|
1402
|
+
attr_list = None
|
|
1403
|
+
for a, t, length in _attrs(frozen):
|
|
1404
|
+
if t == AT_ATTRIBUTE_LIST:
|
|
1405
|
+
attr_list = (frozen, a, length)
|
|
1406
|
+
yield (frozen, a, length)
|
|
1407
|
+
if attr_list is None:
|
|
1408
|
+
return
|
|
1409
|
+
frozen0, a, length = attr_list
|
|
1410
|
+
if frozen0[a + 8] == 0: # resident list value
|
|
1411
|
+
v_len = struct.unpack_from('<I', frozen0, a + 16)[0]
|
|
1412
|
+
v_ofs = struct.unpack_from('<H', frozen0, a + 20)[0]
|
|
1413
|
+
listing = frozen0[a + v_ofs:a + v_ofs + v_len]
|
|
1414
|
+
else:
|
|
1415
|
+
_p, _ph, runs = _check_mapping_pairs(frozen0, a, length, self.nr_clusters())
|
|
1416
|
+
size = struct.unpack_from('<q', frozen0, a + 48)[0]
|
|
1417
|
+
listing = self._runs_read(runs, 0, size)
|
|
1418
|
+
pos, seen = 0, set()
|
|
1419
|
+
while pos + 26 <= len(listing):
|
|
1420
|
+
e_len = struct.unpack_from('<H', listing, pos + 4)[0]
|
|
1421
|
+
if e_len < 26:
|
|
1422
|
+
break
|
|
1423
|
+
mref = struct.unpack_from('<Q', listing, pos + 16)[0] & MREF_MASK
|
|
1424
|
+
if mref != rec_no and mref not in seen:
|
|
1425
|
+
seen.add(mref)
|
|
1426
|
+
ext = self._load_record(mref)
|
|
1427
|
+
if ext is not None:
|
|
1428
|
+
eb = bytes(ext)
|
|
1429
|
+
for ea, _et, el in _attrs(eb):
|
|
1430
|
+
yield (eb, ea, el)
|
|
1431
|
+
pos += e_len
|
|
1432
|
+
|
|
1433
|
+
def _stream_runs(self, rec_no: int, attr_type: int, name: bytes | None):
|
|
1434
|
+
'''(data_size, vcn-ordered runs) of a non-resident attribute, pieces
|
|
1435
|
+
merged across extension records. Raises ENOENT when absent.'''
|
|
1436
|
+
data_size, runs, found = None, [], False
|
|
1437
|
+
for frozen, a, length in self._record_attrs(rec_no):
|
|
1438
|
+
if struct.unpack_from('<I', frozen, a)[0] != attr_type:
|
|
1439
|
+
continue
|
|
1440
|
+
nlen = frozen[a + 9]
|
|
1441
|
+
nofs = struct.unpack_from('<H', frozen, a + 10)[0]
|
|
1442
|
+
aname = frozen[a + nofs:a + nofs + 2 * nlen] if nlen else None
|
|
1443
|
+
if (name or None) != (aname or None):
|
|
1444
|
+
continue
|
|
1445
|
+
found = True
|
|
1446
|
+
if frozen[a + 8] == 0:
|
|
1447
|
+
return None, None # resident — caller reads the value instead
|
|
1448
|
+
if struct.unpack_from('<q', frozen, a + 16)[0] == 0: # lowest_vcn
|
|
1449
|
+
data_size = struct.unpack_from('<q', frozen, a + 48)[0]
|
|
1450
|
+
_p, _ph, r = _check_mapping_pairs(frozen, a, length, self.nr_clusters())
|
|
1451
|
+
runs.extend(r)
|
|
1452
|
+
if not found:
|
|
1453
|
+
raise NtfsError(errno.ENOENT, f'record {rec_no}: attr {attr_type:#x} absent')
|
|
1454
|
+
runs.sort(key=lambda t: t[2])
|
|
1455
|
+
return data_size if data_size is not None else 0, runs
|
|
1456
|
+
|
|
1457
|
+
def _attr_value(self, rec_no: int, attr_type: int, name: bytes | None) -> bytes:
|
|
1458
|
+
for frozen, a, _length in self._record_attrs(rec_no):
|
|
1459
|
+
if struct.unpack_from('<I', frozen, a)[0] != attr_type:
|
|
1460
|
+
continue
|
|
1461
|
+
nlen = frozen[a + 9]
|
|
1462
|
+
nofs = struct.unpack_from('<H', frozen, a + 10)[0]
|
|
1463
|
+
aname = frozen[a + nofs:a + nofs + 2 * nlen] if nlen else None
|
|
1464
|
+
if (name or None) != (aname or None):
|
|
1465
|
+
continue
|
|
1466
|
+
if frozen[a + 8] == 0:
|
|
1467
|
+
v_len = struct.unpack_from('<I', frozen, a + 16)[0]
|
|
1468
|
+
v_ofs = struct.unpack_from('<H', frozen, a + 20)[0]
|
|
1469
|
+
return frozen[a + v_ofs:a + v_ofs + v_len]
|
|
1470
|
+
size, runs = self._stream_runs(rec_no, attr_type, name)
|
|
1471
|
+
return self._runs_read(runs, 0, size)
|
|
1472
|
+
raise NtfsError(errno.ENOENT, f'record {rec_no}: attr {attr_type:#x} absent')
|
|
1473
|
+
|
|
1474
|
+
def _read_whole_attr(self, mft_no: int, attr_type: int,
|
|
1475
|
+
name=None, name_len: int = 0) -> bytes:
|
|
1476
|
+
return self._attr_value(mft_no, attr_type, name)
|
|
1477
|
+
|
|
1478
|
+
# -- directories: raw $I30 walk + path resolution --
|
|
1479
|
+
|
|
1480
|
+
I30 = '$I30'.encode('utf-16-le')
|
|
1481
|
+
|
|
1482
|
+
def _dir_entries(self, mft_no: int) -> list[dict]:
|
|
1483
|
+
root = self._attr_value(mft_no, AT_INDEX_ROOT, self.I30)
|
|
1484
|
+
alloc = bmp = None
|
|
1485
|
+
try:
|
|
1486
|
+
alloc = self._attr_value(mft_no, AT_INDEX_ALLOCATION, self.I30)
|
|
1487
|
+
bmp = self._attr_value(mft_no, AT_BITMAP, self.I30)
|
|
1488
|
+
except NtfsError:
|
|
1489
|
+
pass
|
|
1490
|
+
pairs, problems = _view_index_entries(root, alloc, bmp, f'mft#{mft_no}',
|
|
1491
|
+
dir_index=True)
|
|
1492
|
+
if problems:
|
|
1493
|
+
raise NtfsError(errno.EIO, f'$I30 of record {mft_no}: {problems[0]}')
|
|
1494
|
+
entries = []
|
|
1495
|
+
for key, mref in pairs:
|
|
1496
|
+
if len(key) < 66:
|
|
1497
|
+
continue
|
|
1498
|
+
n_len, n_type = key[64], key[65]
|
|
1499
|
+
attrs = struct.unpack_from('<I', key, 56)[0]
|
|
1500
|
+
entries.append({'name': key[66:66 + 2 * n_len].decode('utf-16-le', 'replace'),
|
|
1501
|
+
'name_type': n_type, 'mref': mref,
|
|
1502
|
+
'dt_type': NTFS_DT_DIR if attrs & 0x10000000 else 8})
|
|
1503
|
+
return entries
|
|
1504
|
+
|
|
1505
|
+
def _resolve(self, path: str) -> int:
|
|
1506
|
+
cur = FILE_ROOT
|
|
1507
|
+
for comp in path.strip('/').split('/'):
|
|
1508
|
+
if not comp:
|
|
1509
|
+
continue
|
|
1510
|
+
for e in self._dir_entries(cur):
|
|
1511
|
+
if self.names_equal(e['name'], comp):
|
|
1512
|
+
cur = e['mref'] & MREF_MASK
|
|
1513
|
+
break
|
|
1514
|
+
else:
|
|
1515
|
+
raise NtfsError(errno.ENOENT, f'{comp!r} not found resolving {path!r}')
|
|
1516
|
+
return cur
|
|
1517
|
+
|
|
1518
|
+
def scan_dir_no(self, mft_no: int) -> list[dict]:
|
|
1519
|
+
entries = self._dir_entries(mft_no)
|
|
1520
|
+
for entry in entries:
|
|
1521
|
+
entry['record'] = entry['mref'] & MREF_MASK
|
|
1522
|
+
entry['seq'] = entry['mref'] >> 48
|
|
1523
|
+
entry['status'] = self._entry_status(mft_no, entry)
|
|
1524
|
+
return entries
|
|
1525
|
+
|
|
1526
|
+
def scan_dir(self, path: str) -> list[dict]:
|
|
1527
|
+
return self.scan_dir_no(self._resolve(path))
|
|
1528
|
+
|
|
1529
|
+
def resolve(self, path: str) -> int:
|
|
1530
|
+
return self._resolve(path)
|
|
1531
|
+
|
|
1532
|
+
def record_info(self, record_no: int) -> dict:
|
|
1533
|
+
try:
|
|
1534
|
+
attrs = list(self._record_attrs(record_no & MREF_MASK))
|
|
1535
|
+
except NtfsError as exc:
|
|
1536
|
+
return {'open': False, 'errno': exc.errno or errno.EIO, 'error': str(exc)}
|
|
1537
|
+
frozen = attrs[0][0] if attrs else self.read_record(record_no & MREF_MASK)
|
|
1538
|
+
info = {'open': True,
|
|
1539
|
+
'seq': struct.unpack_from('<H', frozen, 16)[0],
|
|
1540
|
+
'link_count': struct.unpack_from('<H', frozen, 18)[0],
|
|
1541
|
+
'in_use': bool(struct.unpack_from('<H', frozen, 22)[0] & 1),
|
|
1542
|
+
'is_dir': bool(struct.unpack_from('<H', frozen, 22)[0] & 2),
|
|
1543
|
+
'names': []}
|
|
1544
|
+
for rec, a, _length in attrs:
|
|
1545
|
+
if struct.unpack_from('<I', rec, a)[0] != AT_FILE_NAME or rec[a + 8]:
|
|
1546
|
+
continue
|
|
1547
|
+
v_len = struct.unpack_from('<I', rec, a + 16)[0]
|
|
1548
|
+
v_ofs = struct.unpack_from('<H', rec, a + 20)[0]
|
|
1549
|
+
fn = rec[a + v_ofs:a + v_ofs + v_len]
|
|
1550
|
+
if len(fn) < 66:
|
|
1551
|
+
continue
|
|
1552
|
+
parent = struct.unpack_from('<Q', fn, 0)[0]
|
|
1553
|
+
info['names'].append({
|
|
1554
|
+
'name': fn[66:66 + 2 * fn[64]].decode('utf-16-le', 'replace'),
|
|
1555
|
+
'type': fn[65], 'parent_record': parent & MREF_MASK,
|
|
1556
|
+
'parent_seq': parent >> 48})
|
|
1557
|
+
return info
|
|
1558
|
+
|
|
1559
|
+
def classify_dirent(self, path: str, name: str) -> dict:
|
|
1560
|
+
dir_no = self._resolve(path)
|
|
1561
|
+
for e in self._dir_entries(dir_no):
|
|
1562
|
+
if self.names_equal(e['name'], name):
|
|
1563
|
+
found = {'name': name, 'record': e['mref'] & MREF_MASK,
|
|
1564
|
+
'seq': e['mref'] >> 48}
|
|
1565
|
+
return self._classify_found(dir_no, name, found)
|
|
1566
|
+
raise NtfsError(errno.ENOENT, f'index entry {name!r} not found in {path!r}')
|
|
1567
|
+
|
|
1568
|
+
def remove_dirent(self, path: str, name: str, really: bool) -> dict:
|
|
1569
|
+
if really:
|
|
1570
|
+
raise NtfsError(0, 'RawVolume is read-only — use RawVolumeRW')
|
|
1571
|
+
dir_no = self._resolve(path)
|
|
1572
|
+
for e in self._dir_entries(dir_no):
|
|
1573
|
+
if self.names_equal(e['name'], name):
|
|
1574
|
+
return {'name': name, 'record': e['mref'] & MREF_MASK,
|
|
1575
|
+
'seq': e['mref'] >> 48}
|
|
1576
|
+
raise NtfsError(errno.ENOENT, f'index entry {name!r} not found in {path!r}')
|
|
1577
|
+
|
|
1578
|
+
def usn_check(self, path: str = '/$Extend/$UsnJrnl') -> dict:
|
|
1579
|
+
info = {'present': False, 'problems': [], 'notes': [], 'records': 0,
|
|
1580
|
+
'journal_id': None, 'lowest': None, 'next_usn': None}
|
|
1581
|
+
try:
|
|
1582
|
+
jrnl = self._resolve(path)
|
|
1583
|
+
except NtfsError:
|
|
1584
|
+
return info
|
|
1585
|
+
info['present'] = True
|
|
1586
|
+
problems = info['problems']
|
|
1587
|
+
lowest = 0
|
|
1588
|
+
try:
|
|
1589
|
+
blob = self._attr_value(jrnl, AT_DATA, '$Max'.encode('utf-16-le'))
|
|
1590
|
+
except NtfsError:
|
|
1591
|
+
problems.append('$Max stream missing')
|
|
1592
|
+
blob = None
|
|
1593
|
+
if blob is not None:
|
|
1594
|
+
if len(blob) != 32:
|
|
1595
|
+
problems.append(f'$Max is {len(blob)} bytes, expected 32')
|
|
1596
|
+
else:
|
|
1597
|
+
max_size, _d, jid, lowest = struct.unpack('<QQQQ', blob)
|
|
1598
|
+
info.update(journal_id=jid, lowest=lowest)
|
|
1599
|
+
if not 0 < max_size < 1 << 40:
|
|
1600
|
+
problems.append(f'implausible journal MaximumSize {max_size}')
|
|
1601
|
+
jn = '$J'.encode('utf-16-le')
|
|
1602
|
+
try:
|
|
1603
|
+
data_size, runs = self._stream_runs(jrnl, AT_DATA, jn)
|
|
1604
|
+
if data_size is None: # resident $J
|
|
1605
|
+
value = self._attr_value(jrnl, AT_DATA, jn)
|
|
1606
|
+
data_size, runs = len(value), None
|
|
1607
|
+
except NtfsError:
|
|
1608
|
+
problems.append('$J stream missing')
|
|
1609
|
+
return info
|
|
1610
|
+
info['next_usn'] = data_size
|
|
1611
|
+
if lowest > data_size:
|
|
1612
|
+
problems.append(f'LowestValidUsn {lowest} beyond journal end {data_size}')
|
|
1613
|
+
return info
|
|
1614
|
+
if lowest & 7:
|
|
1615
|
+
problems.append(f'LowestValidUsn {lowest} not 8-byte aligned')
|
|
1616
|
+
return info
|
|
1617
|
+
start = lowest
|
|
1618
|
+
if runs:
|
|
1619
|
+
first_alloc = runs[0][2] * self.cluster_size()
|
|
1620
|
+
if first_alloc > start:
|
|
1621
|
+
info['notes'].append(
|
|
1622
|
+
f'LowestValidUsn {start} lags the first allocated byte '
|
|
1623
|
+
f'{first_alloc} (purged region)')
|
|
1624
|
+
start = first_alloc
|
|
1625
|
+
read = lambda pos, count: self._runs_read(runs, pos, count)
|
|
1626
|
+
else:
|
|
1627
|
+
read = lambda pos, count: value[pos:pos + count]
|
|
1628
|
+
count, walk_problems = _walk_usn_records(read, start, data_size)
|
|
1629
|
+
info['records'] = count
|
|
1630
|
+
problems.extend(walk_problems)
|
|
1631
|
+
return info
|
|
1632
|
+
|
|
1633
|
+
|
|
1634
|
+
class _IndexSchema:
|
|
1635
|
+
'''How one NTFS index type names itself, orders its keys, and frames its
|
|
1636
|
+
entries — the B+ write engine is otherwise generic over it. $I30
|
|
1637
|
+
(directory indexes) is the reference schema; the $Secure view indexes
|
|
1638
|
+
$SII/$SDH will plug in the same way.
|
|
1639
|
+
|
|
1640
|
+
name UTF-16LE attribute name ($I30, $SII, $SDH)
|
|
1641
|
+
collation the COLLATION_* rule id (goes in the $INDEX_ROOT header)
|
|
1642
|
+
sort_key stored-key bytes -> a Python-comparable ordering key
|
|
1643
|
+
build_leaf (value, key_bytes) -> a leaf INDEX_ENTRY
|
|
1644
|
+
|
|
1645
|
+
An INDEX_ENTRY stores its key as `key_length` bytes at offset 16; the
|
|
1646
|
+
collation key is derived from those bytes by `sort_key` (for $I30 the
|
|
1647
|
+
stored key is the whole $FILE_NAME and the collation key is its upcased
|
|
1648
|
+
name; for a view index the stored key is the collation key directly).'''
|
|
1649
|
+
|
|
1650
|
+
__slots__ = ('name', 'collation', 'sort_key', 'build_leaf')
|
|
1651
|
+
|
|
1652
|
+
def __init__(self, name, collation, sort_key, build_leaf):
|
|
1653
|
+
self.name = name
|
|
1654
|
+
self.collation = collation
|
|
1655
|
+
self.sort_key = sort_key
|
|
1656
|
+
self.build_leaf = build_leaf
|
|
1657
|
+
|
|
1658
|
+
@staticmethod
|
|
1659
|
+
def stored_key(entry) -> bytes:
|
|
1660
|
+
return bytes(entry[16:16 + struct.unpack_from('<H', entry, 10)[0]])
|
|
1661
|
+
|
|
1662
|
+
def entry_sort_key(self, entry):
|
|
1663
|
+
return self.sort_key(self.stored_key(entry))
|
|
1664
|
+
|
|
1665
|
+
|
|
1666
|
+
class RawVolumeRW(RawVolume):
|
|
1667
|
+
''' The native write engine: sealed record writes, $MFTMirr sync, dirty-flag
|
|
1668
|
+
lifecycle, allocators, B+ tree index edits (remove / insert with splits /
|
|
1669
|
+
bulk-load rebuild), $Secure and USN repairs '''
|
|
1670
|
+
|
|
1671
|
+
def __init__(self, device: str, base: int = 0):
|
|
1672
|
+
super().__init__(device, readonly=True, base=base)
|
|
1673
|
+
os.close(self._fd)
|
|
1674
|
+
self._fd = os.open(device, os.O_RDWR | _O_BINARY)
|
|
1675
|
+
self._mirror = None
|
|
1676
|
+
self._was_dirty = self._dirty_flag()
|
|
1677
|
+
if not self._was_dirty:
|
|
1678
|
+
self._set_dirty(True) # crash mid-op leaves the volume flagged
|
|
1679
|
+
|
|
1680
|
+
def close(self) -> None:
|
|
1681
|
+
if getattr(self, '_fd', None) is not None and not self._was_dirty:
|
|
1682
|
+
self._set_dirty(False) # clean close: clear only what we set
|
|
1683
|
+
super().close()
|
|
1684
|
+
|
|
1685
|
+
# -- dirty flag write side ($VOLUME_INFORMATION probe lives in RawVolume) --
|
|
1686
|
+
|
|
1687
|
+
def _set_dirty(self, on: bool) -> None:
|
|
1688
|
+
rec, off = self._volume_info()
|
|
1689
|
+
flags = struct.unpack_from('<H', rec, off)[0]
|
|
1690
|
+
struct.pack_into('<H', rec, off, flags | 1 if on else flags & ~1)
|
|
1691
|
+
self.write_record(3, rec)
|
|
1692
|
+
|
|
1693
|
+
# -- write plumbing --
|
|
1694
|
+
|
|
1695
|
+
def _runs_write(self, runs, offset: int, data: bytes) -> None:
|
|
1696
|
+
csz = self._cluster_size
|
|
1697
|
+
end, written = offset + len(data), 0
|
|
1698
|
+
for lcn, run_len, vcn in runs:
|
|
1699
|
+
r_start = vcn * csz
|
|
1700
|
+
if r_start >= end:
|
|
1701
|
+
break
|
|
1702
|
+
r_end = r_start + run_len * csz
|
|
1703
|
+
lo, hi = max(offset, r_start), min(end, r_end)
|
|
1704
|
+
if lo >= hi:
|
|
1705
|
+
continue
|
|
1706
|
+
chunk = data[lo - offset:hi - offset]
|
|
1707
|
+
if _pwrite(self._fd, chunk, self._base + lcn * csz + (lo - r_start)) != len(chunk):
|
|
1708
|
+
raise NtfsError(errno.EIO, f'short write at stream offset {lo}')
|
|
1709
|
+
written += len(chunk)
|
|
1710
|
+
if written != len(data):
|
|
1711
|
+
raise NtfsError(errno.EIO, f'write spans a hole ({written}/{len(data)})')
|
|
1712
|
+
|
|
1713
|
+
def _mirror_runs(self):
|
|
1714
|
+
if self._mirror is None:
|
|
1715
|
+
self._mirror = self._stream_runs(1, AT_DATA, None)[1] # $MFTMirr
|
|
1716
|
+
return self._mirror
|
|
1717
|
+
|
|
1718
|
+
def write_record(self, rec_no: int, logical) -> None:
|
|
1719
|
+
'''Seal fixups over LOGICAL (fixed-up) record content and write it.
|
|
1720
|
+
Records 0-3 are mirrored into $MFTMirr, as the driver verifies.'''
|
|
1721
|
+
rec = bytearray(logical)
|
|
1722
|
+
if rec[:4] != b'FILE' or len(rec) != self._rec_size:
|
|
1723
|
+
raise NtfsError(errno.EINVAL, 'refusing to write a non-FILE record')
|
|
1724
|
+
_seal_fixups(rec)
|
|
1725
|
+
blob = bytes(rec)
|
|
1726
|
+
self._runs_write(self._mft_runs, rec_no * self._rec_size, blob)
|
|
1727
|
+
if rec_no < 4:
|
|
1728
|
+
self._runs_write(self._mirror_runs(), rec_no * self._rec_size, blob)
|
|
1729
|
+
self._blk_cache.clear()
|
|
1730
|
+
self._attr_memo.clear()
|
|
1731
|
+
|
|
1732
|
+
# -- phase 2: allocators ($Bitmap clusters + $MFT record slots) --
|
|
1733
|
+
|
|
1734
|
+
def _bitmap_state(self):
|
|
1735
|
+
if getattr(self, '_bm', None) is None:
|
|
1736
|
+
size, runs = self._stream_runs(FILE_BITMAP, AT_DATA, None)
|
|
1737
|
+
self._bm = bytearray(self._runs_read(runs, 0, size))
|
|
1738
|
+
self._bm_runs = runs
|
|
1739
|
+
return self._bm
|
|
1740
|
+
|
|
1741
|
+
def _bitmap_flush(self, bits) -> None:
|
|
1742
|
+
lo, hi = min(bits) >> 3, (max(bits) >> 3) + 1
|
|
1743
|
+
self._runs_write(self._bm_runs, lo, bytes(self._bm[lo:hi]))
|
|
1744
|
+
|
|
1745
|
+
def apply_bitmap(self, audit: dict) -> None:
|
|
1746
|
+
'''Rewrite $Bitmap with the computed truth (the chkdsk stage-5 fix).'''
|
|
1747
|
+
if audit['failures']:
|
|
1748
|
+
raise NtfsError(0, 'refusing to write $Bitmap: usage map is incomplete')
|
|
1749
|
+
nc = audit['nr_clusters']
|
|
1750
|
+
data = bytearray(audit['used_bytes'])
|
|
1751
|
+
if nc & 7: # preserve the on-disk padding-bit convention in the tail
|
|
1752
|
+
data[-1] |= audit['ondisk_tail'] & (0xFF ^ ((1 << (nc & 7)) - 1))
|
|
1753
|
+
self._bitmap_state() # populate self._bm_runs
|
|
1754
|
+
self._runs_write(self._bm_runs, 0, bytes(data))
|
|
1755
|
+
self._bm = None # drop the cached map; reload lazily
|
|
1756
|
+
|
|
1757
|
+
def secure_fix_mirrors(self, fixes: list) -> int:
|
|
1758
|
+
'''Copy the hash-verified side of each mismatched $SDS entry pair over
|
|
1759
|
+
the corrupt side (both copies end up canonical: offset = primary).'''
|
|
1760
|
+
name = '$SDS'.encode('utf-16-le')
|
|
1761
|
+
size, runs = self._stream_runs(FILE_SECURE, AT_DATA, name)
|
|
1762
|
+
done = 0
|
|
1763
|
+
for pos, length, source in fixes:
|
|
1764
|
+
src = pos if source == 'primary' else pos + SDS_BLOCK
|
|
1765
|
+
blob = bytearray(self._runs_read(runs, src, length))
|
|
1766
|
+
struct.pack_into('<Q', blob, 8, pos) # canonical offset field
|
|
1767
|
+
for dst in (pos, pos + SDS_BLOCK):
|
|
1768
|
+
self._runs_write(runs, dst, bytes(blob))
|
|
1769
|
+
done += 1
|
|
1770
|
+
return done
|
|
1771
|
+
|
|
1772
|
+
def usn_reset(self, path: str = '/$Extend/$UsnJrnl') -> int:
|
|
1773
|
+
'''The chkdsk/fsutil-style journal reset: truncate $J to zero (freeing
|
|
1774
|
+
its clusters) and stamp $Max with a fresh journal id and
|
|
1775
|
+
LowestValidUsn 0. Consumers detect the id change and rescan.
|
|
1776
|
+
Returns the new journal id.'''
|
|
1777
|
+
jrnl = self._resolve(path)
|
|
1778
|
+
jn = '$J'.encode('utf-16-le')
|
|
1779
|
+
found = self._find_attr(jrnl, AT_DATA, jn)
|
|
1780
|
+
if found is None:
|
|
1781
|
+
raise NtfsError(errno.ENOENT, 'open $J')
|
|
1782
|
+
rno, rec, a = found
|
|
1783
|
+
rec = bytearray(rec)
|
|
1784
|
+
if rec[a + 8] == 1: # non-resident: free + zero
|
|
1785
|
+
length = struct.unpack_from('<I', rec, a + 4)[0]
|
|
1786
|
+
_p, _ph, r = _check_mapping_pairs(rec, a, length, self.nr_clusters())
|
|
1787
|
+
runs = [(lcn, rlen) for lcn, rlen, _v in r]
|
|
1788
|
+
struct.pack_into('<q', rec, a + 0x10, 0) # lowest_vcn
|
|
1789
|
+
struct.pack_into('<q', rec, a + 0x18, -1) # highest_vcn (0 clusters)
|
|
1790
|
+
struct.pack_into('<Q', rec, a + 0x28, 0) # allocated_size
|
|
1791
|
+
struct.pack_into('<Q', rec, a + 0x30, 0) # data_size
|
|
1792
|
+
struct.pack_into('<Q', rec, a + 0x38, 0) # initialized_size
|
|
1793
|
+
mp_off = struct.unpack_from('<H', rec, a + 0x20)[0]
|
|
1794
|
+
rec[a + mp_off] = 0 # runlist terminator
|
|
1795
|
+
self.write_record(rno, rec) # clear ref before freeing
|
|
1796
|
+
if runs:
|
|
1797
|
+
self.free_clusters(runs)
|
|
1798
|
+
else: # resident: empty the value
|
|
1799
|
+
self._replace_resident_value(rec, a, b'')
|
|
1800
|
+
self.write_record(rno, rec)
|
|
1801
|
+
|
|
1802
|
+
mn = '$Max'.encode('utf-16-le')
|
|
1803
|
+
try:
|
|
1804
|
+
old = self._attr_value(jrnl, AT_DATA, mn)
|
|
1805
|
+
except NtfsError:
|
|
1806
|
+
old = b''
|
|
1807
|
+
max_size, delta = struct.unpack_from('<QQ', old) if len(old) >= 16 else (0, 0)
|
|
1808
|
+
if not 0 < max_size < 1 << 40 or not 0 < delta < 1 << 40:
|
|
1809
|
+
max_size, delta = 32 * 1024 * 1024, 8 * 1024 * 1024
|
|
1810
|
+
new_id = int((time.time() + 11644473600) * 10 ** 7) # NTFS FILETIME
|
|
1811
|
+
blob = struct.pack('<QQQQ', max_size, delta, new_id, 0)
|
|
1812
|
+
mfound = self._find_attr(jrnl, AT_DATA, mn)
|
|
1813
|
+
if mfound is None:
|
|
1814
|
+
raise NtfsError(errno.ENOENT, 'open $Max')
|
|
1815
|
+
mrno, mrec, ma = mfound
|
|
1816
|
+
mrec = bytearray(mrec)
|
|
1817
|
+
if mrec[ma + 8] == 0: # resident (typical)
|
|
1818
|
+
self._replace_resident_value(mrec, ma, blob)
|
|
1819
|
+
self.write_record(mrno, mrec)
|
|
1820
|
+
else: # non-resident: overwrite
|
|
1821
|
+
_size, mruns = self._stream_runs(jrnl, AT_DATA, mn)
|
|
1822
|
+
self._runs_write(mruns, 0, blob)
|
|
1823
|
+
return new_id
|
|
1824
|
+
|
|
1825
|
+
def rebuild_secure_indexes(self, sds_entries: dict) -> dict:
|
|
1826
|
+
'''Native $SII/$SDH rebuild: wipe each index and re-add one entry per
|
|
1827
|
+
$SDS descriptor. `sds_entries` maps security_id -> (hash, sds_offset,
|
|
1828
|
+
length). Bulk-loads a resident root, a single INDX block, or a
|
|
1829
|
+
multi-level tree as needed (via the shared _bulk_write_index engine),
|
|
1830
|
+
with the two view collations — $SII by security-id (ULONG), $SDH by
|
|
1831
|
+
(hash, id) (SECURITY_HASH).'''
|
|
1832
|
+
orders = (
|
|
1833
|
+
(self._sii, sorted(sds_entries),
|
|
1834
|
+
lambda sid: struct.pack('<I', sid)),
|
|
1835
|
+
(self._sdh, sorted(sds_entries, key=lambda i: (sds_entries[i][0], i)),
|
|
1836
|
+
lambda sid: struct.pack('<II', sds_entries[sid][0], sid)))
|
|
1837
|
+
built = 0
|
|
1838
|
+
for schema, order, key_of in orders:
|
|
1839
|
+
entries = []
|
|
1840
|
+
for sid in order:
|
|
1841
|
+
h, off, length = sds_entries[sid]
|
|
1842
|
+
data = struct.pack('<IIQI', h, sid, off, length)
|
|
1843
|
+
entries.append(schema.build_leaf(data, key_of(sid)))
|
|
1844
|
+
self._write_view_index(FILE_SECURE, schema, entries)
|
|
1845
|
+
built += len(entries)
|
|
1846
|
+
return {'added': built}
|
|
1847
|
+
|
|
1848
|
+
def _write_view_index(self, owner: int, schema, entries: list) -> None:
|
|
1849
|
+
'''Replace `schema`'s index on `owner` with `entries` (already in
|
|
1850
|
+
collation order) — the $SII/$SDH ($Secure) analogue of the $I30
|
|
1851
|
+
rebuild: resident root, single INDX block, or a bulk-loaded
|
|
1852
|
+
multi-level tree, all through the shared _bulk_write_index engine.'''
|
|
1853
|
+
self._ix_override = schema
|
|
1854
|
+
try:
|
|
1855
|
+
self._bulk_write_index(owner, entries)
|
|
1856
|
+
finally:
|
|
1857
|
+
self._ix_override = None
|
|
1858
|
+
|
|
1859
|
+
def alloc_clusters(self, count: int, near_lcn: int = 0) -> list[tuple[int, int]]:
|
|
1860
|
+
'''First-fit from the hint (byte-skipping full bytes), wrapping once.
|
|
1861
|
+
Sets the bits, flushes $Bitmap, returns [(lcn, run_len)] runs. The
|
|
1862
|
+
stage-5 audit is the oracle: freshly allocated, unreferenced
|
|
1863
|
+
clusters must show up as exactly `count` extra bits.'''
|
|
1864
|
+
bm = self._bitmap_state()
|
|
1865
|
+
nc = self.nr_clusters()
|
|
1866
|
+
picked: list[int] = []
|
|
1867
|
+
c = min(max(near_lcn, 0), nc - 1)
|
|
1868
|
+
scanned = 0
|
|
1869
|
+
while len(picked) < count and scanned <= nc:
|
|
1870
|
+
if c >= nc:
|
|
1871
|
+
c = 0
|
|
1872
|
+
if not c & 7 and bm[c >> 3] == 0xFF and c + 8 <= nc:
|
|
1873
|
+
c += 8
|
|
1874
|
+
scanned += 8
|
|
1875
|
+
continue
|
|
1876
|
+
if not bm[c >> 3] & (1 << (c & 7)):
|
|
1877
|
+
picked.append(c)
|
|
1878
|
+
c += 1
|
|
1879
|
+
scanned += 1
|
|
1880
|
+
if len(picked) < count:
|
|
1881
|
+
raise NtfsError(errno.ENOSPC, 'volume full')
|
|
1882
|
+
for c in picked:
|
|
1883
|
+
bm[c >> 3] |= 1 << (c & 7)
|
|
1884
|
+
self._bitmap_flush(picked)
|
|
1885
|
+
picked.sort()
|
|
1886
|
+
runs, start, prev = [], picked[0], picked[0]
|
|
1887
|
+
for c in picked[1:]:
|
|
1888
|
+
if c != prev + 1:
|
|
1889
|
+
runs.append((start, prev - start + 1))
|
|
1890
|
+
start = c
|
|
1891
|
+
prev = c
|
|
1892
|
+
runs.append((start, prev - start + 1))
|
|
1893
|
+
return runs
|
|
1894
|
+
|
|
1895
|
+
def free_clusters(self, runs) -> None:
|
|
1896
|
+
bm = self._bitmap_state()
|
|
1897
|
+
bits, seen = [], set()
|
|
1898
|
+
for lcn, run_len in runs:
|
|
1899
|
+
for c in range(lcn, lcn + run_len):
|
|
1900
|
+
if c in seen or not bm[c >> 3] & (1 << (c & 7)):
|
|
1901
|
+
raise NtfsError(errno.EIO, f'double free of cluster {c}')
|
|
1902
|
+
seen.add(c)
|
|
1903
|
+
bits.append(c)
|
|
1904
|
+
for c in bits:
|
|
1905
|
+
bm[c >> 3] &= ~(1 << (c & 7))
|
|
1906
|
+
self._bitmap_flush(bits)
|
|
1907
|
+
|
|
1908
|
+
def _mft_bitmap(self):
|
|
1909
|
+
'''(bitmap_bytes, writer) for $MFT's own $BITMAP — resident on small
|
|
1910
|
+
volumes (rewrite record 0, mirrored), non-resident on real ones.'''
|
|
1911
|
+
for frozen, a, _l in self._record_attrs(0):
|
|
1912
|
+
if (struct.unpack_from('<I', frozen, a)[0] != AT_BITMAP
|
|
1913
|
+
or frozen[a + 9] != 0):
|
|
1914
|
+
continue # unnamed only
|
|
1915
|
+
if frozen[a + 8] == 0:
|
|
1916
|
+
v_len = struct.unpack_from('<I', frozen, a + 16)[0]
|
|
1917
|
+
v_ofs = struct.unpack_from('<H', frozen, a + 20)[0]
|
|
1918
|
+
blob = frozen[a + v_ofs:a + v_ofs + v_len]
|
|
1919
|
+
|
|
1920
|
+
def write_res(data: bytes, a=a, v_ofs=v_ofs, v_len=v_len):
|
|
1921
|
+
rec = self._load_record(0)
|
|
1922
|
+
assert rec is not None
|
|
1923
|
+
rec[a + v_ofs:a + v_ofs + v_len] = data
|
|
1924
|
+
self.write_record(0, rec)
|
|
1925
|
+
return blob, write_res
|
|
1926
|
+
_p, _ph, runs = _check_mapping_pairs(frozen, a,
|
|
1927
|
+
struct.unpack_from('<I', frozen,
|
|
1928
|
+
a + 4)[0],
|
|
1929
|
+
self.nr_clusters())
|
|
1930
|
+
size = struct.unpack_from('<q', frozen, a + 48)[0]
|
|
1931
|
+
blob = self._runs_read(runs, 0, size)
|
|
1932
|
+
|
|
1933
|
+
def write_nonres(data: bytes, runs=runs):
|
|
1934
|
+
self._runs_write(runs, 0, data)
|
|
1935
|
+
return blob, write_nonres
|
|
1936
|
+
raise NtfsError(errno.ENOENT, "$MFT's $BITMAP not found")
|
|
1937
|
+
|
|
1938
|
+
def alloc_record(self) -> int:
|
|
1939
|
+
'''Reuse a free MFT record slot (bit clear AND record not in use);
|
|
1940
|
+
growing the MFT is deliberately unimplemented — ENOSPC instead.'''
|
|
1941
|
+
bm, write = self._mft_bitmap()
|
|
1942
|
+
new = bytearray(bm)
|
|
1943
|
+
for rec_no in range(24, self._mft_size // self._rec_size):
|
|
1944
|
+
if new[rec_no >> 3] & (1 << (rec_no & 7)):
|
|
1945
|
+
continue
|
|
1946
|
+
rec = self._load_record(rec_no)
|
|
1947
|
+
if rec is not None and struct.unpack_from('<H', rec, 22)[0] & 1:
|
|
1948
|
+
continue # bitmap stale: record actually live — never take it
|
|
1949
|
+
new[rec_no >> 3] |= 1 << (rec_no & 7)
|
|
1950
|
+
write(bytes(new))
|
|
1951
|
+
return rec_no
|
|
1952
|
+
raise NtfsError(errno.ENOSPC, 'no free MFT records (growth unimplemented)')
|
|
1953
|
+
|
|
1954
|
+
def apply_mft_bitmap(self, used: bytes, nbits: int) -> int:
|
|
1955
|
+
'''Rewrite $MFT's $BITMAP so bit i matches record i's surveyed in-use
|
|
1956
|
+
flag, for records [0, nbits); bits past nbits keep their on-disk
|
|
1957
|
+
value. The crash shape this repairs: alloc_record's bitmap write
|
|
1958
|
+
reached the disk but the record write never did — a leaked set bit
|
|
1959
|
+
that chkdsk reports as "the MFT BITMAP attribute is incorrect".'''
|
|
1960
|
+
bm, write = self._mft_bitmap()
|
|
1961
|
+
new = bytearray(bm)
|
|
1962
|
+
changed = 0
|
|
1963
|
+
for i in range(min(nbits, len(new) * 8)):
|
|
1964
|
+
want = bool(used[i >> 3] & (1 << (i & 7)))
|
|
1965
|
+
if bool(new[i >> 3] & (1 << (i & 7))) != want:
|
|
1966
|
+
changed += 1
|
|
1967
|
+
if want:
|
|
1968
|
+
new[i >> 3] |= 1 << (i & 7)
|
|
1969
|
+
else:
|
|
1970
|
+
new[i >> 3] &= ~(1 << (i & 7)) & 0xFF
|
|
1971
|
+
if changed:
|
|
1972
|
+
write(bytes(new))
|
|
1973
|
+
return changed
|
|
1974
|
+
|
|
1975
|
+
def free_record(self, rec_no: int) -> None:
|
|
1976
|
+
bm, write = self._mft_bitmap()
|
|
1977
|
+
if not bm[rec_no >> 3] & (1 << (rec_no & 7)):
|
|
1978
|
+
raise NtfsError(errno.EIO, f'record {rec_no} already free in bitmap')
|
|
1979
|
+
new = bytearray(bm)
|
|
1980
|
+
new[rec_no >> 3] &= ~(1 << (rec_no & 7))
|
|
1981
|
+
write(bytes(new))
|
|
1982
|
+
|
|
1983
|
+
# -- phase 3/3b: index-entry removal (leaf splice + internal promotion) --
|
|
1984
|
+
|
|
1985
|
+
def _write_index_block(self, ia_runs, block_index: int, block_size: int,
|
|
1986
|
+
blk: bytearray) -> None:
|
|
1987
|
+
_seal_fixups(blk)
|
|
1988
|
+
self._runs_write(ia_runs, block_index * block_size, bytes(blk))
|
|
1989
|
+
self._blk_cache.clear()
|
|
1990
|
+
self._attr_memo.clear()
|
|
1991
|
+
|
|
1992
|
+
@staticmethod
|
|
1993
|
+
def _find_entry(buf, hdr_off: int, matches):
|
|
1994
|
+
'''(pos, length, flags) of the first non-END entry for which
|
|
1995
|
+
matches(name) is true, else None. Does not mutate.'''
|
|
1996
|
+
entries_ofs, index_len = struct.unpack_from('<II', buf, hdr_off)
|
|
1997
|
+
pos, limit = hdr_off + entries_ofs, hdr_off + index_len
|
|
1998
|
+
while pos + 16 <= limit:
|
|
1999
|
+
length = struct.unpack_from('<H', buf, pos + 8)[0]
|
|
2000
|
+
flags = struct.unpack_from('<H', buf, pos + 12)[0]
|
|
2001
|
+
if length < 16 or pos + length > limit:
|
|
2002
|
+
return None
|
|
2003
|
+
if not flags & 2:
|
|
2004
|
+
n_len = buf[pos + 16 + 64]
|
|
2005
|
+
nm = bytes(buf[pos + 16 + 66:pos + 16 + 66 + 2 * n_len]).decode(
|
|
2006
|
+
'utf-16-le', 'replace')
|
|
2007
|
+
if matches(nm):
|
|
2008
|
+
return pos, length, flags
|
|
2009
|
+
if flags & 2:
|
|
2010
|
+
break
|
|
2011
|
+
pos += length
|
|
2012
|
+
return None
|
|
2013
|
+
|
|
2014
|
+
|
|
2015
|
+
@staticmethod
|
|
2016
|
+
def _splice_at(buf, hdr_off: int, pos: int, length: int) -> None:
|
|
2017
|
+
''' Remove the entry at pos '''
|
|
2018
|
+
|
|
2019
|
+
_eo, index_len = struct.unpack_from('<II', buf, hdr_off)
|
|
2020
|
+
limit = hdr_off + index_len
|
|
2021
|
+
tail = bytes(buf[pos + length:limit])
|
|
2022
|
+
buf[pos:pos + len(tail)] = tail
|
|
2023
|
+
new_len = index_len - length
|
|
2024
|
+
for i in range(hdr_off + new_len, limit):
|
|
2025
|
+
buf[i] = 0
|
|
2026
|
+
struct.pack_into('<I', buf, hdr_off + 4, new_len)
|
|
2027
|
+
|
|
2028
|
+
|
|
2029
|
+
def _read_indx_by_vcn(self, alloc, block_size, vcn):
|
|
2030
|
+
'''(block_index, fixed-up block) whose header VCN == vcn, matched by the
|
|
2031
|
+
block's own index_block_vcn field (VCN-unit-agnostic) '''
|
|
2032
|
+
|
|
2033
|
+
for i in range(len(alloc) // block_size):
|
|
2034
|
+
if alloc[i * block_size:i * block_size + 4] != b'INDX':
|
|
2035
|
+
continue
|
|
2036
|
+
if struct.unpack_from('<q', alloc, i * block_size + 16)[0] != vcn:
|
|
2037
|
+
continue
|
|
2038
|
+
blk = bytearray(alloc[i * block_size:(i + 1) * block_size])
|
|
2039
|
+
if _apply_fixups(blk):
|
|
2040
|
+
return i, blk
|
|
2041
|
+
return None, None
|
|
2042
|
+
|
|
2043
|
+
|
|
2044
|
+
def _rightmost_leaf(self, alloc, block_size, start_vcn):
|
|
2045
|
+
'''Descend the rightmost (END-entry) subnode pointers from start_vcn to
|
|
2046
|
+
a leaf block — the subtree's maximum lives there.'''
|
|
2047
|
+
vcn, seen = start_vcn, set()
|
|
2048
|
+
while True:
|
|
2049
|
+
if vcn in seen:
|
|
2050
|
+
raise NtfsError(errno.EIO, 'cyclic index subnode chain')
|
|
2051
|
+
seen.add(vcn)
|
|
2052
|
+
i, blk = self._read_indx_by_vcn(alloc, block_size, vcn)
|
|
2053
|
+
if blk is None:
|
|
2054
|
+
raise NtfsError(errno.EIO, f'index block VCN {vcn} unreadable')
|
|
2055
|
+
eo, il = struct.unpack_from('<II', blk, 24)
|
|
2056
|
+
pos, end = 24 + eo, 24 + il
|
|
2057
|
+
while pos + 16 <= end:
|
|
2058
|
+
length = struct.unpack_from('<H', blk, pos + 8)[0]
|
|
2059
|
+
flags = struct.unpack_from('<H', blk, pos + 12)[0]
|
|
2060
|
+
if length < 16:
|
|
2061
|
+
raise NtfsError(errno.EIO, 'corrupt index node')
|
|
2062
|
+
if flags & 2: # END entry
|
|
2063
|
+
if flags & 1: # has a subnode — descend rightmost
|
|
2064
|
+
vcn = struct.unpack_from('<q', blk, pos + length - 8)[0]
|
|
2065
|
+
break
|
|
2066
|
+
return i, blk # leaf
|
|
2067
|
+
pos += length
|
|
2068
|
+
else:
|
|
2069
|
+
raise NtfsError(errno.EIO, 'index node without END entry')
|
|
2070
|
+
|
|
2071
|
+
|
|
2072
|
+
@staticmethod
|
|
2073
|
+
def _last_real_entry(buf, hdr_off):
|
|
2074
|
+
'''(pos, length) of the last non-END entry — the node's maximum key.'''
|
|
2075
|
+
eo, il = struct.unpack_from('<II', buf, hdr_off)
|
|
2076
|
+
pos, limit, last = hdr_off + eo, hdr_off + il, None
|
|
2077
|
+
while pos + 16 <= limit:
|
|
2078
|
+
length = struct.unpack_from('<H', buf, pos + 8)[0]
|
|
2079
|
+
flags = struct.unpack_from('<H', buf, pos + 12)[0]
|
|
2080
|
+
if length < 16:
|
|
2081
|
+
break
|
|
2082
|
+
if flags & 2:
|
|
2083
|
+
break
|
|
2084
|
+
last = (pos, length)
|
|
2085
|
+
pos += length
|
|
2086
|
+
return last
|
|
2087
|
+
|
|
2088
|
+
|
|
2089
|
+
def _remove_internal(self, node, hdr_off, ie_pos, ie_len, write_node,
|
|
2090
|
+
ia_runs, block_size, alloc) -> None:
|
|
2091
|
+
'''Delete an internal (subnode-bearing) entry by promoting its in-order
|
|
2092
|
+
predecessor — the maximum key of its left subtree — into its slot,
|
|
2093
|
+
then removing that predecessor from its leaf. Faithful to a B-tree
|
|
2094
|
+
internal delete; no rebalancing (underfull nodes stay valid).
|
|
2095
|
+
|
|
2096
|
+
Order matters for crash safety: remove the predecessor from the leaf
|
|
2097
|
+
FIRST, then promote. A crash between the two leaves the target entry
|
|
2098
|
+
in place (repair simply re-runs) and turns the predecessor into a
|
|
2099
|
+
reconnectable lost file — never a duplicate key.'''
|
|
2100
|
+
ie_vcn = struct.unpack_from('<q', node, ie_pos + ie_len - 8)[0]
|
|
2101
|
+
leaf_i, leaf = self._rightmost_leaf(alloc, block_size, ie_vcn)
|
|
2102
|
+
pred = self._last_real_entry(leaf, 24)
|
|
2103
|
+
if pred is None:
|
|
2104
|
+
# legal shape (removals emptied the subtree) that predecessor
|
|
2105
|
+
# promotion cannot cross — the caller rebuilds instead
|
|
2106
|
+
raise NtfsError(errno.ENOTSUP, 'empty subtree under an internal entry')
|
|
2107
|
+
p_pos, p_len = pred
|
|
2108
|
+
p_bytes = bytes(leaf[p_pos:p_pos + p_len])
|
|
2109
|
+
p_flags = struct.unpack_from('<H', p_bytes, 12)[0]
|
|
2110
|
+
if p_flags & 1:
|
|
2111
|
+
raise NtfsError(errno.ENOTSUP, 'predecessor unexpectedly has a subnode')
|
|
2112
|
+
|
|
2113
|
+
# build the promoted separator: predecessor key/data + the deleted
|
|
2114
|
+
# entry's own subnode VCN, marked as a NODE entry
|
|
2115
|
+
key_len = struct.unpack_from('<H', p_bytes, 10)[0]
|
|
2116
|
+
content = p_bytes[:16 + key_len]
|
|
2117
|
+
content += b'\x00' * (-len(content) % 8) # 8-byte align
|
|
2118
|
+
repl = bytearray(content + struct.pack('<q', ie_vcn))
|
|
2119
|
+
struct.pack_into('<H', repl, 8, len(repl))
|
|
2120
|
+
struct.pack_into('<H', repl, 12, (p_flags | 1) & ~2) # NODE, not END
|
|
2121
|
+
|
|
2122
|
+
eo, index_len = struct.unpack_from('<II', node, hdr_off)
|
|
2123
|
+
alloc_size = struct.unpack_from('<I', node, hdr_off + 8)[0]
|
|
2124
|
+
new_index_len = index_len - ie_len + len(repl)
|
|
2125
|
+
if new_index_len > alloc_size:
|
|
2126
|
+
raise NtfsError(errno.ENOTSUP,
|
|
2127
|
+
'promoted separator does not fit — node split needed')
|
|
2128
|
+
|
|
2129
|
+
# 1) drop the predecessor from its leaf (crash-safe half)
|
|
2130
|
+
self._splice_at(leaf, 24, p_pos, p_len)
|
|
2131
|
+
self._write_index_block(ia_runs, leaf_i, block_size, leaf)
|
|
2132
|
+
# 2) replace the target entry with the promoted separator
|
|
2133
|
+
tail = bytes(node[ie_pos + ie_len:hdr_off + index_len])
|
|
2134
|
+
node[ie_pos:ie_pos + len(repl)] = repl
|
|
2135
|
+
node[ie_pos + len(repl):ie_pos + len(repl) + len(tail)] = tail
|
|
2136
|
+
for i in range(hdr_off + new_index_len, hdr_off + index_len):
|
|
2137
|
+
node[i] = 0
|
|
2138
|
+
struct.pack_into('<I', node, hdr_off + 4, new_index_len)
|
|
2139
|
+
write_node()
|
|
2140
|
+
|
|
2141
|
+
def remove_index_entry(self, dir_no: int, name: str) -> bool:
|
|
2142
|
+
'''Remove one $I30 entry by name — leaf via splice, internal via
|
|
2143
|
+
predecessor promotion; shapes the promotion cannot cross (emptied
|
|
2144
|
+
subtree, overfull node) fall back to a bulk rebuild of the index
|
|
2145
|
+
from its own surviving entries. Raises ENOENT when absent.'''
|
|
2146
|
+
try:
|
|
2147
|
+
return self._remove_index_entry_structural(dir_no, name)
|
|
2148
|
+
except NtfsError as exc:
|
|
2149
|
+
if exc.errno != errno.ENOTSUP:
|
|
2150
|
+
raise
|
|
2151
|
+
self._rebuild_without(dir_no, name)
|
|
2152
|
+
return True
|
|
2153
|
+
|
|
2154
|
+
def _rebuild_without(self, dir_no: int, name: str) -> None:
|
|
2155
|
+
'''Bulk-rebuild the index from its own current entries minus `name` —
|
|
2156
|
+
always yields a canonical tree, reclaiming emptied blocks.'''
|
|
2157
|
+
root = self._attr_value(dir_no, AT_INDEX_ROOT, self._ix.name)
|
|
2158
|
+
try:
|
|
2159
|
+
alloc = self._attr_value(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
|
|
2160
|
+
except NtfsError:
|
|
2161
|
+
alloc = b''
|
|
2162
|
+
block_size = struct.unpack_from('<I', root, 8)[0]
|
|
2163
|
+
nodes = [(bytes(root), 16)]
|
|
2164
|
+
for i in range(len(alloc) // block_size):
|
|
2165
|
+
blk = bytearray(alloc[i * block_size:(i + 1) * block_size])
|
|
2166
|
+
if blk[:4] == b'INDX' and _apply_fixups(blk):
|
|
2167
|
+
nodes.append((bytes(blk), 24))
|
|
2168
|
+
items = []
|
|
2169
|
+
for buf, hdr in nodes:
|
|
2170
|
+
reals, _end = self._decode_node(buf, hdr)
|
|
2171
|
+
for e in reals:
|
|
2172
|
+
klen = struct.unpack_from('<H', e, 10)[0]
|
|
2173
|
+
fn = e[16:16 + klen]
|
|
2174
|
+
nm = self._fn_key_name(fn).decode('utf-16-le', 'replace')
|
|
2175
|
+
if self.names_equal(nm, name):
|
|
2176
|
+
continue
|
|
2177
|
+
mref = struct.unpack_from('<Q', e, 0)[0]
|
|
2178
|
+
leaf = self._ix.build_leaf(mref, fn)
|
|
2179
|
+
items.append((self._ix.entry_sort_key(leaf), leaf))
|
|
2180
|
+
items.sort(key=lambda t: t[0])
|
|
2181
|
+
self._bulk_write_index(dir_no, [e for _, e in items])
|
|
2182
|
+
|
|
2183
|
+
def _remove_index_entry_structural(self, dir_no: int, name: str) -> bool:
|
|
2184
|
+
rec = self._load_record(dir_no)
|
|
2185
|
+
if rec is None:
|
|
2186
|
+
raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
|
|
2187
|
+
loc = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
|
|
2188
|
+
if loc is None:
|
|
2189
|
+
raise NtfsError(errno.ENOENT, f'record {dir_no} has no $I30 index root')
|
|
2190
|
+
a, _ln = loc
|
|
2191
|
+
vofs = struct.unpack_from('<H', rec, a + 20)[0]
|
|
2192
|
+
root_vofs = a + vofs
|
|
2193
|
+
block_size = struct.unpack_from('<I', rec, a + vofs + 8)[0]
|
|
2194
|
+
|
|
2195
|
+
alloc = ia_runs = None
|
|
2196
|
+
|
|
2197
|
+
def _index_alloc():
|
|
2198
|
+
nonlocal alloc, ia_runs
|
|
2199
|
+
if alloc is None:
|
|
2200
|
+
alloc = self._attr_value(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
|
|
2201
|
+
_, ia_runs = self._stream_runs(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
|
|
2202
|
+
return alloc, ia_runs
|
|
2203
|
+
|
|
2204
|
+
match = lambda nm: self.names_equal(nm, name)
|
|
2205
|
+
ent = self._find_entry(rec, root_vofs + 16, match)
|
|
2206
|
+
if ent:
|
|
2207
|
+
pos, length, flags = ent
|
|
2208
|
+
if not flags & 1: # leaf entry in the root
|
|
2209
|
+
self._splice_at(rec, root_vofs + 16, pos, length)
|
|
2210
|
+
self.write_record(dir_no, rec)
|
|
2211
|
+
else: # internal entry in the root
|
|
2212
|
+
al, runs = _index_alloc()
|
|
2213
|
+
self._remove_internal(rec, root_vofs + 16, pos, length,
|
|
2214
|
+
lambda: self.write_record(dir_no, rec),
|
|
2215
|
+
runs, block_size, al)
|
|
2216
|
+
return True
|
|
2217
|
+
|
|
2218
|
+
try:
|
|
2219
|
+
al, runs = _index_alloc()
|
|
2220
|
+
except NtfsError:
|
|
2221
|
+
raise NtfsError(errno.ENOENT, f'{name!r} not found in directory {dir_no}')
|
|
2222
|
+
for i in range(len(al) // block_size):
|
|
2223
|
+
blk = bytearray(al[i * block_size:(i + 1) * block_size])
|
|
2224
|
+
if blk[:4] != b'INDX' or not _apply_fixups(blk):
|
|
2225
|
+
continue
|
|
2226
|
+
ent = self._find_entry(blk, 24, match)
|
|
2227
|
+
if not ent:
|
|
2228
|
+
continue
|
|
2229
|
+
pos, length, flags = ent
|
|
2230
|
+
if not flags & 1: # leaf entry in an INDX block
|
|
2231
|
+
self._splice_at(blk, 24, pos, length)
|
|
2232
|
+
self._write_index_block(runs, i, block_size, blk)
|
|
2233
|
+
else: # internal entry in an INDX block
|
|
2234
|
+
def _write(bi=i, bb=blk):
|
|
2235
|
+
self._write_index_block(runs, bi, block_size, bb)
|
|
2236
|
+
self._remove_internal(blk, 24, pos, length, _write,
|
|
2237
|
+
runs, block_size, al)
|
|
2238
|
+
return True
|
|
2239
|
+
raise NtfsError(errno.ENOENT, f'{name!r} not found in directory {dir_no}')
|
|
2240
|
+
|
|
2241
|
+
def remove_dirent(self, path: str, name: str, really: bool) -> dict:
|
|
2242
|
+
info = super().remove_dirent(path, name, really=False) # name, record, seq
|
|
2243
|
+
if not really:
|
|
2244
|
+
return info
|
|
2245
|
+
self.remove_index_entry(self._resolve(path), name)
|
|
2246
|
+
info['removed'] = True
|
|
2247
|
+
return info
|
|
2248
|
+
|
|
2249
|
+
# -- phase 4: index-entry insertion (leaf insert + root grow; split deferred) --
|
|
2250
|
+
|
|
2251
|
+
@staticmethod
|
|
2252
|
+
def _fn_key_name(fn_bytes) -> bytes:
|
|
2253
|
+
return bytes(fn_bytes[66:66 + 2 * fn_bytes[64]])
|
|
2254
|
+
|
|
2255
|
+
def _upcase_seq(self, name_u16: bytes):
|
|
2256
|
+
'''UTF-16LE name upcased through $UpCase as a tuple of code units —
|
|
2257
|
+
the collation key. memoryview.cast avoids struct.unpack, and the
|
|
2258
|
+
full 65536-entry table needs no per-unit bounds check.'''
|
|
2259
|
+
vals = memoryview(name_u16).cast('H')
|
|
2260
|
+
up = self._upcase_table()
|
|
2261
|
+
if up is None:
|
|
2262
|
+
return tuple(vals)
|
|
2263
|
+
if len(up) >= 0x10000:
|
|
2264
|
+
return tuple([up[v] for v in vals])
|
|
2265
|
+
n = len(up)
|
|
2266
|
+
return tuple([up[v] if v < n else v for v in vals])
|
|
2267
|
+
|
|
2268
|
+
@staticmethod
|
|
2269
|
+
def _build_leaf_entry(mref: int, fn_bytes: bytes) -> bytes:
|
|
2270
|
+
klen = len(fn_bytes)
|
|
2271
|
+
elen = (16 + klen + 7) & ~7
|
|
2272
|
+
e = bytearray(elen)
|
|
2273
|
+
struct.pack_into('<QHHH', e, 0, mref, elen, klen, 0) # ref,len,keylen,flags
|
|
2274
|
+
e[16:16 + klen] = fn_bytes
|
|
2275
|
+
return bytes(e)
|
|
2276
|
+
|
|
2277
|
+
@property
|
|
2278
|
+
def _i30(self) -> _IndexSchema:
|
|
2279
|
+
'''The $I30 directory-index schema (COLLATION_FILE_NAME): the stored key
|
|
2280
|
+
is the whole $FILE_NAME, ordered by its upcased name; the entry value
|
|
2281
|
+
is the 8-byte MFT reference.'''
|
|
2282
|
+
s = self.__dict__.get('_i30_cache')
|
|
2283
|
+
if s is None:
|
|
2284
|
+
s = self.__dict__['_i30_cache'] = _IndexSchema(
|
|
2285
|
+
name=self.I30, collation=0x01,
|
|
2286
|
+
sort_key=lambda fn: self._upcase_seq(fn[66:66 + 2 * fn[64]]),
|
|
2287
|
+
build_leaf=self._build_leaf_entry)
|
|
2288
|
+
return s
|
|
2289
|
+
|
|
2290
|
+
@staticmethod
|
|
2291
|
+
def _build_view_leaf(data: bytes, key: bytes, magic: bool = False) -> bytes:
|
|
2292
|
+
'''A leaf INDEX_ENTRY for a $Secure view index: {data_offset, data_length}
|
|
2293
|
+
header, the key at offset 16, then the 20-byte SDS locator as the
|
|
2294
|
+
entry data. $SDH carries a 4-byte "II" (UTF-16LE) magic after the
|
|
2295
|
+
data; $SII does not.'''
|
|
2296
|
+
klen = len(key)
|
|
2297
|
+
doff = 16 + klen
|
|
2298
|
+
tail = b'\x49\x00\x49\x00' if magic else b''
|
|
2299
|
+
elen = (doff + len(data) + len(tail) + 7) & ~7
|
|
2300
|
+
e = bytearray(elen)
|
|
2301
|
+
struct.pack_into('<HH', e, 0, doff, len(data)) # data_offset, data_length
|
|
2302
|
+
struct.pack_into('<HHH', e, 8, elen, klen, 0) # entry_len, key_len, flags
|
|
2303
|
+
e[16:16 + klen] = key
|
|
2304
|
+
e[doff:doff + len(data)] = data
|
|
2305
|
+
e[doff + len(data):doff + len(data) + len(tail)] = tail
|
|
2306
|
+
return bytes(e)
|
|
2307
|
+
|
|
2308
|
+
@property
|
|
2309
|
+
def _sii(self) -> _IndexSchema:
|
|
2310
|
+
'''$Secure $SII: key = security_id (u32), COLLATION_NTOFS_ULONG.'''
|
|
2311
|
+
s = self.__dict__.get('_sii_cache')
|
|
2312
|
+
if s is None:
|
|
2313
|
+
s = self.__dict__['_sii_cache'] = _IndexSchema(
|
|
2314
|
+
name='$SII'.encode('utf-16-le'), collation=0x10,
|
|
2315
|
+
sort_key=lambda k: int.from_bytes(k[:4], 'little'),
|
|
2316
|
+
build_leaf=lambda data, key: self._build_view_leaf(data, key))
|
|
2317
|
+
return s
|
|
2318
|
+
|
|
2319
|
+
@property
|
|
2320
|
+
def _sdh(self) -> _IndexSchema:
|
|
2321
|
+
'''$Secure $SDH: key = {hash u32, id u32}, COLLATION_NTOFS_SECURITY_HASH.'''
|
|
2322
|
+
s = self.__dict__.get('_sdh_cache')
|
|
2323
|
+
if s is None:
|
|
2324
|
+
s = self.__dict__['_sdh_cache'] = _IndexSchema(
|
|
2325
|
+
name='$SDH'.encode('utf-16-le'), collation=0x12,
|
|
2326
|
+
sort_key=lambda k: struct.unpack_from('<II', k),
|
|
2327
|
+
build_leaf=lambda data, key: self._build_view_leaf(data, key, magic=True))
|
|
2328
|
+
return s
|
|
2329
|
+
|
|
2330
|
+
@property
|
|
2331
|
+
def _ix(self) -> _IndexSchema:
|
|
2332
|
+
'''The index schema the B+ write engine currently operates on — $I30 by
|
|
2333
|
+
default. (A generic view-index insert will scope self._ix_override.)'''
|
|
2334
|
+
return getattr(self, '_ix_override', None) or self._i30
|
|
2335
|
+
|
|
2336
|
+
def _resident_grow_root(self, rec: bytearray, root_attr_off: int,
|
|
2337
|
+
delta: int) -> None:
|
|
2338
|
+
'''Resize the resident $INDEX_ROOT attribute by delta bytes in place
|
|
2339
|
+
(negative shrinks), shifting the attributes after it and the
|
|
2340
|
+
record's used size. Refuses if the record has no room for a grow
|
|
2341
|
+
(that needs small→large conversion).'''
|
|
2342
|
+
in_use = struct.unpack_from('<I', rec, 0x18)[0]
|
|
2343
|
+
if in_use + delta > len(rec):
|
|
2344
|
+
raise NtfsError(errno.ENOTSUP,
|
|
2345
|
+
'directory record full — cannot grow the resident index root')
|
|
2346
|
+
attr_len = struct.unpack_from('<I', rec, root_attr_off + 4)[0]
|
|
2347
|
+
tail = bytes(rec[root_attr_off + attr_len:in_use])
|
|
2348
|
+
rec[root_attr_off + attr_len + delta:
|
|
2349
|
+
root_attr_off + attr_len + delta + len(tail)] = tail
|
|
2350
|
+
for i in range(root_attr_off + attr_len, root_attr_off + attr_len + delta):
|
|
2351
|
+
rec[i] = 0 # grow: zero the inserted gap
|
|
2352
|
+
for i in range(in_use + delta, in_use):
|
|
2353
|
+
rec[i] = 0 # shrink: zero the freed tail
|
|
2354
|
+
vlen = struct.unpack_from('<I', rec, root_attr_off + 0x10)[0]
|
|
2355
|
+
vofs = struct.unpack_from('<H', rec, root_attr_off + 0x14)[0]
|
|
2356
|
+
struct.pack_into('<I', rec, root_attr_off + 4, attr_len + delta)
|
|
2357
|
+
struct.pack_into('<I', rec, root_attr_off + 0x10, vlen + delta)
|
|
2358
|
+
struct.pack_into('<I', rec, 0x18, in_use + delta)
|
|
2359
|
+
hdr = root_attr_off + vofs + 16 # INDEX_HEADER.allocated_size
|
|
2360
|
+
asz = struct.unpack_from('<I', rec, hdr + 8)[0]
|
|
2361
|
+
struct.pack_into('<I', rec, hdr + 8, asz + delta)
|
|
2362
|
+
|
|
2363
|
+
# -- phase 4d: attribute creation + small→large conversion --
|
|
2364
|
+
|
|
2365
|
+
def _add_attr(self, rec: bytearray, attr_type: int, name_u16: bytes,
|
|
2366
|
+
resident: bool, value: bytes = b'', runs=None,
|
|
2367
|
+
data_size: int = 0) -> int:
|
|
2368
|
+
'''Build an attribute and insert it into the base record in (type, name)
|
|
2369
|
+
order. resident: uses `value`. non-resident: uses `runs` (encoded as
|
|
2370
|
+
mapping pairs) + data_size. Returns the attribute's record offset;
|
|
2371
|
+
raises ENOTSUP if the record is full ($ATTRIBUTE_LIST spill needed).'''
|
|
2372
|
+
name_len = len(name_u16) // 2
|
|
2373
|
+
if resident:
|
|
2374
|
+
name_off = 0x18
|
|
2375
|
+
val_off = ((name_off + len(name_u16) + 7) & ~7) if name_len else 0x18
|
|
2376
|
+
alen = (val_off + len(value) + 7) & ~7
|
|
2377
|
+
attr = bytearray(alen)
|
|
2378
|
+
struct.pack_into('<IIBBH', attr, 0, attr_type, alen, 0, name_len,
|
|
2379
|
+
name_off if name_len else 0)
|
|
2380
|
+
struct.pack_into('<IH', attr, 0x10, len(value), val_off)
|
|
2381
|
+
if name_len:
|
|
2382
|
+
attr[name_off:name_off + len(name_u16)] = name_u16
|
|
2383
|
+
attr[val_off:val_off + len(value)] = value
|
|
2384
|
+
else:
|
|
2385
|
+
name_off = 0x40
|
|
2386
|
+
mp_off = ((name_off + len(name_u16) + 7) & ~7) if name_len else 0x40
|
|
2387
|
+
mp = _encode_mapping_pairs(runs)
|
|
2388
|
+
alen = (mp_off + len(mp) + 7) & ~7
|
|
2389
|
+
total = sum(n for _lcn, n in runs)
|
|
2390
|
+
attr = bytearray(alen)
|
|
2391
|
+
struct.pack_into('<IIBBH', attr, 0, attr_type, alen, 1, name_len,
|
|
2392
|
+
name_off if name_len else 0)
|
|
2393
|
+
struct.pack_into('<qq', attr, 0x10, 0, total - 1) # lowest/highest vcn
|
|
2394
|
+
struct.pack_into('<H', attr, 0x20, mp_off)
|
|
2395
|
+
struct.pack_into('<qqq', attr, 0x28, data_size, data_size, data_size)
|
|
2396
|
+
if name_len:
|
|
2397
|
+
attr[name_off:name_off + len(name_u16)] = name_u16
|
|
2398
|
+
attr[mp_off:mp_off + len(mp)] = mp
|
|
2399
|
+
inst = struct.unpack_from('<H', rec, 0x28)[0]
|
|
2400
|
+
struct.pack_into('<H', attr, 0x0E, inst)
|
|
2401
|
+
struct.pack_into('<H', rec, 0x28, inst + 1)
|
|
2402
|
+
|
|
2403
|
+
in_use = struct.unpack_from('<I', rec, 0x18)[0]
|
|
2404
|
+
if in_use + alen > len(rec):
|
|
2405
|
+
raise NtfsError(errno.ENOTSUP, 'record full — $ATTRIBUTE_LIST spill needed')
|
|
2406
|
+
a = struct.unpack_from('<H', rec, 0x14)[0]
|
|
2407
|
+
while a + 4 <= in_use: # (type, name) order
|
|
2408
|
+
t = struct.unpack_from('<I', rec, a)[0]
|
|
2409
|
+
if t == 0xFFFFFFFF or t > attr_type:
|
|
2410
|
+
break
|
|
2411
|
+
a += struct.unpack_from('<I', rec, a + 4)[0]
|
|
2412
|
+
tail = bytes(rec[a:in_use])
|
|
2413
|
+
rec[a + alen:a + alen + len(tail)] = tail
|
|
2414
|
+
rec[a:a + alen] = attr
|
|
2415
|
+
struct.pack_into('<I', rec, 0x18, in_use + alen)
|
|
2416
|
+
return a
|
|
2417
|
+
|
|
2418
|
+
def _replace_resident_value(self, rec: bytearray, attr_off: int,
|
|
2419
|
+
new_value: bytes) -> None:
|
|
2420
|
+
'''Replace a resident attribute's value (any length), shifting the
|
|
2421
|
+
attributes after it and the record's used size. Shrinking frees
|
|
2422
|
+
record space; growing needs room (else the caller must spill).'''
|
|
2423
|
+
vofs = struct.unpack_from('<H', rec, attr_off + 0x14)[0]
|
|
2424
|
+
old_alen = struct.unpack_from('<I', rec, attr_off + 4)[0]
|
|
2425
|
+
new_alen = (vofs + len(new_value) + 7) & ~7
|
|
2426
|
+
in_use = struct.unpack_from('<I', rec, 0x18)[0]
|
|
2427
|
+
if in_use + (new_alen - old_alen) > len(rec):
|
|
2428
|
+
raise NtfsError(errno.ENOTSUP, 'record full')
|
|
2429
|
+
tail = bytes(rec[attr_off + old_alen:in_use])
|
|
2430
|
+
rec[attr_off + new_alen:attr_off + new_alen + len(tail)] = tail
|
|
2431
|
+
rec[attr_off + vofs:attr_off + vofs + len(new_value)] = new_value
|
|
2432
|
+
struct.pack_into('<I', rec, attr_off + 4, new_alen)
|
|
2433
|
+
struct.pack_into('<I', rec, attr_off + 0x10, len(new_value))
|
|
2434
|
+
new_in_use = in_use + (new_alen - old_alen)
|
|
2435
|
+
struct.pack_into('<I', rec, 0x18, new_in_use)
|
|
2436
|
+
for i in range(new_in_use, in_use): # zero any freed tail
|
|
2437
|
+
rec[i] = 0
|
|
2438
|
+
|
|
2439
|
+
def _small_to_large(self, dir_no: int, block_size: int, vpb: int) -> None:
|
|
2440
|
+
'''Convert a SMALL_INDEX (resident $INDEX_ROOT only) into a LARGE index:
|
|
2441
|
+
shrink the root to a single END→block pointer (freeing record space),
|
|
2442
|
+
create the $INDEX_ALLOCATION and $BITMAP attributes in that space,
|
|
2443
|
+
and move the root's entries into the new block.'''
|
|
2444
|
+
rec = self._load_record(dir_no)
|
|
2445
|
+
if rec is None:
|
|
2446
|
+
raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
|
|
2447
|
+
ra, _l = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
|
|
2448
|
+
rvofs = ra + struct.unpack_from('<H', rec, ra + 0x14)[0]
|
|
2449
|
+
reals, end = self._decode_node(rec, rvofs + 16) # leaf entries + END
|
|
2450
|
+
# 1) shrink the root to a minimal LARGE root: prefix + header + END→0
|
|
2451
|
+
prefix = bytes(rec[rvofs:rvofs + 16])
|
|
2452
|
+
end_node = self._end_node(0)
|
|
2453
|
+
vlen = 32 + len(end_node)
|
|
2454
|
+
ih = struct.pack('<IIIBxxx', 16, 16 + len(end_node), vlen - 16, 1)
|
|
2455
|
+
self._replace_resident_value(rec, ra, prefix + ih + end_node)
|
|
2456
|
+
# 2) create $INDEX_ALLOCATION (1 block) and $BITMAP in the freed space
|
|
2457
|
+
cpb = block_size // self._cluster_size
|
|
2458
|
+
runs = self.alloc_clusters(cpb)
|
|
2459
|
+
try:
|
|
2460
|
+
self._add_attr(rec, AT_INDEX_ALLOCATION, self._ix.name, resident=False,
|
|
2461
|
+
runs=[(l, n) for l, n in runs], data_size=block_size)
|
|
2462
|
+
bm = bytearray(8) # 1 bit/block; 8 = 64 blocks
|
|
2463
|
+
bm[0] = 1 # block 0 in use
|
|
2464
|
+
self._add_attr(rec, AT_BITMAP, self._ix.name, resident=True, value=bytes(bm))
|
|
2465
|
+
except NtfsError:
|
|
2466
|
+
self.free_clusters(runs)
|
|
2467
|
+
raise
|
|
2468
|
+
self.write_record(dir_no, rec)
|
|
2469
|
+
if getattr(self, '_ia_cache', None):
|
|
2470
|
+
self._ia_cache.pop(dir_no, None)
|
|
2471
|
+
# 3) move the old root entries into block 0
|
|
2472
|
+
blk = self._new_indx_block(0, block_size)
|
|
2473
|
+
self._pack_node(blk, 24, reals, end)
|
|
2474
|
+
self._write_block(dir_no, 0, block_size, blk)
|
|
2475
|
+
|
|
2476
|
+
# -- phase 4b: node split — insertion never refuses (large indexes) --
|
|
2477
|
+
|
|
2478
|
+
@staticmethod
|
|
2479
|
+
def _leaf_end() -> bytes:
|
|
2480
|
+
return struct.pack('<QHHHH', 0, 16, 0, 2, 0) # END, no subnode
|
|
2481
|
+
|
|
2482
|
+
@staticmethod
|
|
2483
|
+
def _end_node(vcn: int) -> bytes:
|
|
2484
|
+
return struct.pack('<QHHHH', 0, 24, 0, 3, 0) + struct.pack('<q', vcn)
|
|
2485
|
+
|
|
2486
|
+
@staticmethod
|
|
2487
|
+
def _entry_subnode(e: bytes) -> int:
|
|
2488
|
+
return struct.unpack_from('<q', e, len(e) - 8)[0]
|
|
2489
|
+
|
|
2490
|
+
@staticmethod
|
|
2491
|
+
def _set_subnode(e: bytes, vcn: int) -> bytes:
|
|
2492
|
+
b = bytearray(e)
|
|
2493
|
+
struct.pack_into('<q', b, len(b) - 8, vcn)
|
|
2494
|
+
return bytes(b)
|
|
2495
|
+
|
|
2496
|
+
@staticmethod
|
|
2497
|
+
def _make_node_entry(e: bytes, vcn: int) -> bytes:
|
|
2498
|
+
'''Promote entry e to a NODE separator: the entry followed by the
|
|
2499
|
+
8-byte subnode VCN. A $Secure view-index entry carries its 20-byte
|
|
2500
|
+
data (+ $SDH magic) into the separator — NTFS keeps the median's data
|
|
2501
|
+
in the internal node — while a directory ($I30) entry is just
|
|
2502
|
+
header+key+pad, so this is byte-identical to the old header+key form
|
|
2503
|
+
there. An entry that is already a separator (multi-level split)
|
|
2504
|
+
must first shed its own subnode VCN, or the ghost 8 bytes make
|
|
2505
|
+
entry_length non-canonical — chkdsk flags that as an index error.'''
|
|
2506
|
+
elen = struct.unpack_from('<H', e, 8)[0]
|
|
2507
|
+
flags = struct.unpack_from('<H', e, 12)[0]
|
|
2508
|
+
if flags & 1:
|
|
2509
|
+
elen -= 8
|
|
2510
|
+
base = bytes(e[:elen])
|
|
2511
|
+
base += b'\x00' * (-len(base) % 8)
|
|
2512
|
+
b = bytearray(base + struct.pack('<q', vcn))
|
|
2513
|
+
struct.pack_into('<H', b, 8, len(b))
|
|
2514
|
+
struct.pack_into('<H', b, 12, 1) # NODE, not END
|
|
2515
|
+
return bytes(b)
|
|
2516
|
+
|
|
2517
|
+
@staticmethod
|
|
2518
|
+
def _decode_node(buf, hdr_off):
|
|
2519
|
+
eo, il = struct.unpack_from('<II', buf, hdr_off)
|
|
2520
|
+
pos, limit, reals, end = hdr_off + eo, hdr_off + il, [], None
|
|
2521
|
+
while pos + 16 <= limit:
|
|
2522
|
+
length = struct.unpack_from('<H', buf, pos + 8)[0]
|
|
2523
|
+
flags = struct.unpack_from('<H', buf, pos + 12)[0]
|
|
2524
|
+
if length < 16:
|
|
2525
|
+
break
|
|
2526
|
+
e = bytes(buf[pos:pos + length])
|
|
2527
|
+
if flags & 2:
|
|
2528
|
+
end = e
|
|
2529
|
+
break
|
|
2530
|
+
reals.append(e)
|
|
2531
|
+
pos += length
|
|
2532
|
+
if end is None:
|
|
2533
|
+
raise NtfsError(errno.EIO, 'index node without END entry')
|
|
2534
|
+
return reals, end
|
|
2535
|
+
|
|
2536
|
+
@staticmethod
|
|
2537
|
+
def _pack_node(buf, hdr_off, reals, end) -> bool:
|
|
2538
|
+
'''Write reals+end into the node; False if it overflows allocated_size.
|
|
2539
|
+
Also fixes the INDEX_HEADER node flag from the END entry.'''
|
|
2540
|
+
eo = struct.unpack_from('<I', buf, hdr_off)[0]
|
|
2541
|
+
asz = struct.unpack_from('<I', buf, hdr_off + 8)[0]
|
|
2542
|
+
old_il = struct.unpack_from('<I', buf, hdr_off + 4)[0]
|
|
2543
|
+
body = b''.join(reals) + end
|
|
2544
|
+
new_il = eo + len(body)
|
|
2545
|
+
if new_il > asz:
|
|
2546
|
+
return False
|
|
2547
|
+
p = hdr_off + eo
|
|
2548
|
+
buf[p:p + len(body)] = body
|
|
2549
|
+
for i in range(p + len(body), hdr_off + max(old_il, new_il)):
|
|
2550
|
+
buf[i] = 0
|
|
2551
|
+
struct.pack_into('<I', buf, hdr_off + 4, new_il)
|
|
2552
|
+
node_flag = struct.unpack_from('<H', end, 12)[0] & 1
|
|
2553
|
+
buf[hdr_off + 12] = node_flag # INDEX_HEADER.flags: has children?
|
|
2554
|
+
return True
|
|
2555
|
+
|
|
2556
|
+
def _new_indx_block(self, vcn: int, block_size: int):
|
|
2557
|
+
blk = bytearray(block_size)
|
|
2558
|
+
blk[:4] = b'INDX'
|
|
2559
|
+
usa_ofs, usa_count = 40, 1 + block_size // 512
|
|
2560
|
+
struct.pack_into('<HH', blk, 4, usa_ofs, usa_count)
|
|
2561
|
+
struct.pack_into('<q', blk, 16, vcn)
|
|
2562
|
+
entries_off = ((usa_ofs + 2 * usa_count + 7) & ~7) - 24
|
|
2563
|
+
struct.pack_into('<IIIB', blk, 24, entries_off, entries_off,
|
|
2564
|
+
block_size - 24, 0)
|
|
2565
|
+
return blk
|
|
2566
|
+
|
|
2567
|
+
def _ia(self, dir_no):
|
|
2568
|
+
'''Cached (size, runs) of $INDEX_ALLOCATION — invalidated only on grow.'''
|
|
2569
|
+
cache = getattr(self, '_ia_cache', None)
|
|
2570
|
+
if cache is None:
|
|
2571
|
+
cache = self._ia_cache = {}
|
|
2572
|
+
got = cache.get(dir_no)
|
|
2573
|
+
if got is None:
|
|
2574
|
+
got = cache[dir_no] = self._stream_runs(dir_no, AT_INDEX_ALLOCATION,
|
|
2575
|
+
self._ix.name)
|
|
2576
|
+
return got
|
|
2577
|
+
|
|
2578
|
+
def _read_block(self, dir_no, block_index, block_size):
|
|
2579
|
+
_size, runs = self._ia(dir_no)
|
|
2580
|
+
return bytearray(self._runs_read(runs, block_index * block_size, block_size))
|
|
2581
|
+
|
|
2582
|
+
def _write_block(self, dir_no, block_index, block_size, blk) -> None:
|
|
2583
|
+
_s, runs = self._ia(dir_no)
|
|
2584
|
+
self._write_index_block(runs, block_index, block_size, blk)
|
|
2585
|
+
|
|
2586
|
+
def _alloc_index_block(self, dir_no, block_size, vpb):
|
|
2587
|
+
'''(block_index, vcn) of a free INDX block — reusing a clear $BITMAP
|
|
2588
|
+
bit, else growing $INDEX_ALLOCATION by one block.'''
|
|
2589
|
+
bm = self._read_i30_bitmap(dir_no)
|
|
2590
|
+
size, _runs = self._ia(dir_no)
|
|
2591
|
+
n_blocks = size // block_size
|
|
2592
|
+
for i in range(n_blocks):
|
|
2593
|
+
if not bm[i >> 3] & (1 << (i & 7)):
|
|
2594
|
+
bm[i >> 3] |= 1 << (i & 7)
|
|
2595
|
+
self._write_i30_bitmap(dir_no, bm)
|
|
2596
|
+
return i, i * vpb
|
|
2597
|
+
return self._grow_index_alloc(dir_no, block_size, vpb, bm, n_blocks)
|
|
2598
|
+
|
|
2599
|
+
def _base_attr(self, rec, atype, name):
|
|
2600
|
+
for a, t, ln in _attrs(rec):
|
|
2601
|
+
if t == atype:
|
|
2602
|
+
nlen = rec[a + 9]
|
|
2603
|
+
nofs = struct.unpack_from('<H', rec, a + 10)[0]
|
|
2604
|
+
aname = bytes(rec[a + nofs:a + nofs + 2 * nlen]) if nlen else b''
|
|
2605
|
+
if aname == (name or b''):
|
|
2606
|
+
return a, ln
|
|
2607
|
+
return None
|
|
2608
|
+
|
|
2609
|
+
def _read_i30_bitmap(self, dir_no):
|
|
2610
|
+
found = self._find_attr(dir_no, AT_BITMAP, self._ix.name) # base or extension
|
|
2611
|
+
if not found:
|
|
2612
|
+
raise NtfsError(errno.ENOENT, 'directory has no $I30 $BITMAP')
|
|
2613
|
+
_no, rec, a = found
|
|
2614
|
+
ln = struct.unpack_from('<I', rec, a + 4)[0]
|
|
2615
|
+
if rec[a + 8] == 0:
|
|
2616
|
+
vo = struct.unpack_from('<H', rec, a + 20)[0]
|
|
2617
|
+
vl = struct.unpack_from('<I', rec, a + 16)[0]
|
|
2618
|
+
return bytearray(rec[a + vo:a + vo + vl])
|
|
2619
|
+
_p, _ph, runs = _check_mapping_pairs(bytes(rec), a, ln, self.nr_clusters())
|
|
2620
|
+
return bytearray(self._runs_read(runs, 0, struct.unpack_from('<q', rec, a + 48)[0]))
|
|
2621
|
+
|
|
2622
|
+
def _write_i30_bitmap(self, dir_no, bm):
|
|
2623
|
+
rec_no, rec, a = self._find_attr(dir_no, AT_BITMAP, self._ix.name)
|
|
2624
|
+
ln = struct.unpack_from('<I', rec, a + 4)[0]
|
|
2625
|
+
if rec[a + 8] == 0:
|
|
2626
|
+
vo = struct.unpack_from('<H', rec, a + 20)[0]
|
|
2627
|
+
vl = struct.unpack_from('<I', rec, a + 16)[0]
|
|
2628
|
+
rec[a + vo:a + vo + vl] = bytes(bm)[:vl]
|
|
2629
|
+
self.write_record(rec_no, rec)
|
|
2630
|
+
else:
|
|
2631
|
+
_p, _ph, runs = _check_mapping_pairs(bytes(rec), a, ln, self.nr_clusters())
|
|
2632
|
+
self._runs_write(runs, 0, bytes(bm))
|
|
2633
|
+
|
|
2634
|
+
def _grow_i30_bitmap(self, dir_no, want_bytes: int):
|
|
2635
|
+
'''Extend the $I30 $BITMAP to at least want_bytes. If its record is full,
|
|
2636
|
+
spill $INDEX_ALLOCATION + $BITMAP to an extension record (only when
|
|
2637
|
+
the bitmap still lives in the base) and retry there.'''
|
|
2638
|
+
rec_no, rec, a = self._find_attr(dir_no, AT_BITMAP, self._ix.name)
|
|
2639
|
+
if rec[a + 8] != 0:
|
|
2640
|
+
raise NtfsError(errno.ENOTSUP, 'non-resident $I30 $BITMAP growth not implemented')
|
|
2641
|
+
vo = struct.unpack_from('<H', rec, a + 0x14)[0]
|
|
2642
|
+
vl = struct.unpack_from('<I', rec, a + 0x10)[0]
|
|
2643
|
+
if want_bytes <= vl:
|
|
2644
|
+
return
|
|
2645
|
+
cur = bytes(rec[a + vo:a + vo + vl])
|
|
2646
|
+
try:
|
|
2647
|
+
self._replace_resident_value(rec, a, cur + b'\x00' * (want_bytes - vl))
|
|
2648
|
+
except NtfsError: # record full
|
|
2649
|
+
if rec_no != dir_no:
|
|
2650
|
+
raise NtfsError(errno.ENOTSUP, 'extension $BITMAP record full — '
|
|
2651
|
+
'further spill not implemented')
|
|
2652
|
+
self._spill_index_alloc(dir_no) # frees the base (or re-raises)
|
|
2653
|
+
return self._grow_i30_bitmap(dir_no, want_bytes)
|
|
2654
|
+
self.write_record(rec_no, rec)
|
|
2655
|
+
|
|
2656
|
+
# -- phase 4e: $ATTRIBUTE_LIST spill (base record full → extension records) --
|
|
2657
|
+
|
|
2658
|
+
def _find_attr(self, dir_no, attr_type, name):
|
|
2659
|
+
'''(record_no, fixed-up record, attr_off) of an attribute — searching the
|
|
2660
|
+
base record, then extension records via $ATTRIBUTE_LIST. None if
|
|
2661
|
+
absent. name is UTF-16LE (or b'' for unnamed).'''
|
|
2662
|
+
base = self._load_record(dir_no)
|
|
2663
|
+
if base is None:
|
|
2664
|
+
raise NtfsError(errno.EIO, f'record {dir_no} unreadable')
|
|
2665
|
+
loc = self._base_attr(base, attr_type, name)
|
|
2666
|
+
if loc:
|
|
2667
|
+
return dir_no, base, loc[0]
|
|
2668
|
+
for ext_no in self._extension_records(dir_no):
|
|
2669
|
+
ext = self._load_record(ext_no)
|
|
2670
|
+
if ext is None:
|
|
2671
|
+
continue
|
|
2672
|
+
loc = self._base_attr(ext, attr_type, name)
|
|
2673
|
+
if loc:
|
|
2674
|
+
return ext_no, ext, loc[0]
|
|
2675
|
+
return None
|
|
2676
|
+
|
|
2677
|
+
def _build_extension_record(self, base_no, base_seq, ext_seq, attrs):
|
|
2678
|
+
'''A minimal extension FILE record holding `attrs` (each re-instanced),
|
|
2679
|
+
with base_reference pointing back at the base. Returns the record.'''
|
|
2680
|
+
rs = self._rec_size
|
|
2681
|
+
rec = bytearray(rs)
|
|
2682
|
+
rec[:4] = b'FILE'
|
|
2683
|
+
usa_ofs, usa_count = 0x30, 1 + rs // 512
|
|
2684
|
+
struct.pack_into('<HH', rec, 4, usa_ofs, usa_count)
|
|
2685
|
+
struct.pack_into('<H', rec, 0x10, ext_seq) # sequence
|
|
2686
|
+
struct.pack_into('<H', rec, 0x12, 0) # hard_link_count
|
|
2687
|
+
attrs_off = (usa_ofs + 2 * usa_count + 7) & ~7
|
|
2688
|
+
struct.pack_into('<H', rec, 0x14, attrs_off) # attrs offset
|
|
2689
|
+
struct.pack_into('<H', rec, 0x16, 1) # flags: in use
|
|
2690
|
+
struct.pack_into('<Q', rec, 0x20, base_no | (base_seq << 48))
|
|
2691
|
+
off = attrs_off
|
|
2692
|
+
for inst, attr in enumerate(attrs):
|
|
2693
|
+
a = bytearray(attr)
|
|
2694
|
+
struct.pack_into('<H', a, 0x0E, inst) # instance id
|
|
2695
|
+
rec[off:off + len(a)] = a
|
|
2696
|
+
off += len(a)
|
|
2697
|
+
struct.pack_into('<I', rec, off, 0xFFFFFFFF)
|
|
2698
|
+
struct.pack_into('<I', rec, 0x18, off + 4) # bytes_in_use
|
|
2699
|
+
struct.pack_into('<I', rec, 0x1C, rs) # bytes_allocated
|
|
2700
|
+
struct.pack_into('<H', rec, 0x28, len(attrs)) # next_attr_id
|
|
2701
|
+
struct.pack_into('<I', rec, 0x2C, 0) # record number (unused)
|
|
2702
|
+
return rec
|
|
2703
|
+
|
|
2704
|
+
@staticmethod
|
|
2705
|
+
def _al_entry(atype, name_u16, vcn, ref_no, ref_seq, inst):
|
|
2706
|
+
nl = len(name_u16) // 2
|
|
2707
|
+
rlen = (0x1A + len(name_u16) + 7) & ~7
|
|
2708
|
+
e = bytearray(rlen)
|
|
2709
|
+
struct.pack_into('<IHBB', e, 0, atype, rlen, nl, 0x1A if nl else 0)
|
|
2710
|
+
struct.pack_into('<Q', e, 8, vcn)
|
|
2711
|
+
struct.pack_into('<Q', e, 16, ref_no | (ref_seq << 48))
|
|
2712
|
+
struct.pack_into('<H', e, 24, inst)
|
|
2713
|
+
if nl:
|
|
2714
|
+
e[0x1A:0x1A + len(name_u16)] = name_u16
|
|
2715
|
+
return atype, name_u16, bytes(e)
|
|
2716
|
+
|
|
2717
|
+
def _spill_index_alloc(self, dir_no):
|
|
2718
|
+
'''Move $INDEX_ALLOCATION and $BITMAP out of a full base record into a
|
|
2719
|
+
fresh extension record, adding an $ATTRIBUTE_LIST to the base so both
|
|
2720
|
+
can keep growing in the roomy extension. First spill only (a base
|
|
2721
|
+
that already has a list is refused). Atomic: the extension
|
|
2722
|
+
record is freed if anything fails.'''
|
|
2723
|
+
base = self._load_record(dir_no)
|
|
2724
|
+
if base is None:
|
|
2725
|
+
raise NtfsError(errno.EIO, f'record {dir_no} unreadable')
|
|
2726
|
+
if self._base_attr(base, AT_ATTRIBUTE_LIST, b''):
|
|
2727
|
+
raise NtfsError(errno.ENOTSUP, 'directory already has an '
|
|
2728
|
+
'$ATTRIBUTE_LIST — further spill not implemented')
|
|
2729
|
+
base_seq = struct.unpack_from('<H', base, 0x10)[0]
|
|
2730
|
+
# attributes to relocate, by record offset (removed high→low)
|
|
2731
|
+
move = []
|
|
2732
|
+
for atype in (AT_INDEX_ALLOCATION, AT_BITMAP):
|
|
2733
|
+
loc = self._base_attr(base, atype, self._ix.name)
|
|
2734
|
+
if loc:
|
|
2735
|
+
a = loc[0]
|
|
2736
|
+
alen = struct.unpack_from('<I', base, a + 4)[0]
|
|
2737
|
+
move.append((a, atype, bytes(base[a:a + alen])))
|
|
2738
|
+
if not any(m[1] == AT_INDEX_ALLOCATION for m in move):
|
|
2739
|
+
raise NtfsError(errno.ENOTSUP, '$INDEX_ALLOCATION not in base record')
|
|
2740
|
+
|
|
2741
|
+
ext_no = self.alloc_record()
|
|
2742
|
+
try:
|
|
2743
|
+
old = self.read_record(ext_no)
|
|
2744
|
+
ext_seq = 1
|
|
2745
|
+
if old and old[:4] == b'FILE':
|
|
2746
|
+
ext_seq = (struct.unpack_from('<H', old, 0x10)[0] + 1) & 0xFFFF or 1
|
|
2747
|
+
moved_bytes = [b for _a, _t, b in move]
|
|
2748
|
+
ext = self._build_extension_record(dir_no, base_seq, ext_seq, moved_bytes)
|
|
2749
|
+
ext_inst = {t: i for i, (_a, t, _b) in enumerate(move)} # instance in ext
|
|
2750
|
+
|
|
2751
|
+
# remove the moved attributes from the base (high offset first)
|
|
2752
|
+
for a, _t, blob in sorted(move, key=lambda m: -m[0]):
|
|
2753
|
+
in_use = struct.unpack_from('<I', base, 0x18)[0]
|
|
2754
|
+
alen = len(blob)
|
|
2755
|
+
tail = bytes(base[a + alen:in_use])
|
|
2756
|
+
base[a:a + len(tail)] = tail
|
|
2757
|
+
struct.pack_into('<I', base, 0x18, in_use - alen)
|
|
2758
|
+
for i in range(in_use - alen, in_use):
|
|
2759
|
+
base[i] = 0
|
|
2760
|
+
|
|
2761
|
+
# $ATTRIBUTE_LIST: remaining base attrs → base, moved → extension
|
|
2762
|
+
entries, off = [], struct.unpack_from('<H', base, 0x14)[0]
|
|
2763
|
+
end = struct.unpack_from('<I', base, 0x18)[0]
|
|
2764
|
+
while off + 4 <= end:
|
|
2765
|
+
t = struct.unpack_from('<I', base, off)[0]
|
|
2766
|
+
if t == 0xFFFFFFFF:
|
|
2767
|
+
break
|
|
2768
|
+
ln = struct.unpack_from('<I', base, off + 4)[0]
|
|
2769
|
+
nl = base[off + 9]
|
|
2770
|
+
nofs = struct.unpack_from('<H', base, off + 10)[0]
|
|
2771
|
+
nm = bytes(base[off + nofs:off + nofs + 2 * nl]) if nl else b''
|
|
2772
|
+
inst = struct.unpack_from('<H', base, off + 0x0E)[0]
|
|
2773
|
+
entries.append(self._al_entry(t, nm, 0, dir_no, base_seq, inst))
|
|
2774
|
+
off += ln
|
|
2775
|
+
for _a, t, _b in move:
|
|
2776
|
+
entries.append(self._al_entry(t, self._ix.name, 0, ext_no, ext_seq,
|
|
2777
|
+
ext_inst[t]))
|
|
2778
|
+
entries.sort(key=lambda x: (x[0], x[1]))
|
|
2779
|
+
al_value = b''.join(e for _t, _n, e in entries)
|
|
2780
|
+
self._add_attr(base, AT_ATTRIBUTE_LIST, b'', resident=True, value=al_value)
|
|
2781
|
+
|
|
2782
|
+
self.write_record(ext_no, ext)
|
|
2783
|
+
self.write_record(dir_no, base)
|
|
2784
|
+
except NtfsError:
|
|
2785
|
+
self.free_record(ext_no) # undo the extension allocation
|
|
2786
|
+
raise
|
|
2787
|
+
if getattr(self, '_ia_cache', None):
|
|
2788
|
+
self._ia_cache.pop(dir_no, None)
|
|
2789
|
+
if getattr(self, '_attr_memo', None):
|
|
2790
|
+
self._attr_memo.clear()
|
|
2791
|
+
|
|
2792
|
+
def _grow_index_alloc(self, dir_no, block_size, vpb, bm, n_blocks):
|
|
2793
|
+
'''Append one INDX block to $INDEX_ALLOCATION and set its $BITMAP bit,
|
|
2794
|
+
growing the resident $BITMAP if needed. When the $INDEX_ALLOCATION
|
|
2795
|
+
mapping pairs would overflow the base record, spill it to an
|
|
2796
|
+
extension record (via $ATTRIBUTE_LIST) and retry.'''
|
|
2797
|
+
if (n_blocks >> 3) >= len(bm):
|
|
2798
|
+
self._grow_i30_bitmap(dir_no, (n_blocks // 8 + 8) & ~7)
|
|
2799
|
+
bm = self._read_i30_bitmap(dir_no)
|
|
2800
|
+
cpb = block_size // self._cluster_size
|
|
2801
|
+
found = self._find_attr(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
|
|
2802
|
+
if not found:
|
|
2803
|
+
raise NtfsError(errno.EIO, '$INDEX_ALLOCATION not found')
|
|
2804
|
+
ia_no, rec, a = found
|
|
2805
|
+
ln = struct.unpack_from('<I', rec, a + 4)[0]
|
|
2806
|
+
_p, _ph, existing = _check_mapping_pairs(bytes(rec), a, ln, self.nr_clusters())
|
|
2807
|
+
merged = [(l, n) for l, n, _v in existing]
|
|
2808
|
+
hint = merged[-1][0] + merged[-1][1] if merged else 0
|
|
2809
|
+
new_runs = self.alloc_clusters(cpb, near_lcn=hint)
|
|
2810
|
+
for lcn, rl in new_runs:
|
|
2811
|
+
if merged and merged[-1][0] + merged[-1][1] == lcn:
|
|
2812
|
+
merged[-1] = (merged[-1][0], merged[-1][1] + rl)
|
|
2813
|
+
else:
|
|
2814
|
+
merged.append((lcn, rl))
|
|
2815
|
+
mp = _encode_mapping_pairs(merged)
|
|
2816
|
+
mp_off = struct.unpack_from('<H', rec, a + 32)[0]
|
|
2817
|
+
if mp_off + len(mp) > ln:
|
|
2818
|
+
# the attribute is sized tightly, so a runlist that fragments
|
|
2819
|
+
# routinely needs a few more bytes: grow it in place while its
|
|
2820
|
+
# record has room, spilling only when the record is truly full
|
|
2821
|
+
delta = (mp_off + len(mp) - ln + 7) & ~7
|
|
2822
|
+
in_use = struct.unpack_from('<I', rec, 0x18)[0]
|
|
2823
|
+
if in_use + delta <= len(rec):
|
|
2824
|
+
tail = bytes(rec[a + ln:in_use])
|
|
2825
|
+
rec[a + ln + delta:a + ln + delta + len(tail)] = tail
|
|
2826
|
+
for i in range(a + ln, a + ln + delta):
|
|
2827
|
+
rec[i] = 0
|
|
2828
|
+
struct.pack_into('<I', rec, a + 4, ln + delta)
|
|
2829
|
+
struct.pack_into('<I', rec, 0x18, in_use + delta)
|
|
2830
|
+
ln += delta
|
|
2831
|
+
elif ia_no == dir_no: # still in the base — spill and retry
|
|
2832
|
+
self.free_clusters(new_runs)
|
|
2833
|
+
self._spill_index_alloc(dir_no)
|
|
2834
|
+
return self._grow_index_alloc(dir_no, block_size, vpb,
|
|
2835
|
+
self._read_i30_bitmap(dir_no), n_blocks)
|
|
2836
|
+
else:
|
|
2837
|
+
self.free_clusters(new_runs)
|
|
2838
|
+
raise NtfsError(errno.ENOTSUP, 'extension $INDEX_ALLOCATION record '
|
|
2839
|
+
'full — further spill not implemented')
|
|
2840
|
+
for i in range(a + mp_off, a + ln):
|
|
2841
|
+
rec[i] = 0
|
|
2842
|
+
rec[a + mp_off:a + mp_off + len(mp)] = mp
|
|
2843
|
+
new_size = (n_blocks + 1) * block_size
|
|
2844
|
+
struct.pack_into('<q', rec, a + 24, (n_blocks + 1) * vpb - 1) # highest_vcn
|
|
2845
|
+
struct.pack_into('<q', rec, a + 40, new_size) # allocated
|
|
2846
|
+
struct.pack_into('<q', rec, a + 48, new_size) # data
|
|
2847
|
+
struct.pack_into('<q', rec, a + 56, new_size) # initialized
|
|
2848
|
+
self.write_record(ia_no, rec) # base or extension
|
|
2849
|
+
bm[n_blocks >> 3] |= 1 << (n_blocks & 7)
|
|
2850
|
+
self._write_i30_bitmap(dir_no, bm)
|
|
2851
|
+
getattr(self, '_ia_cache', {}).pop(dir_no, None)
|
|
2852
|
+
return n_blocks, n_blocks * vpb
|
|
2853
|
+
|
|
2854
|
+
def _node_buf(self, dir_no, ref, block_size):
|
|
2855
|
+
'''(buf, hdr_off, root_attr_off|None) for a node reference.'''
|
|
2856
|
+
if ref[0] == 'root':
|
|
2857
|
+
rec = self._load_record(dir_no)
|
|
2858
|
+
if rec is None:
|
|
2859
|
+
raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
|
|
2860
|
+
a, _ln = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
|
|
2861
|
+
vofs = struct.unpack_from('<H', rec, a + 20)[0]
|
|
2862
|
+
return rec, a + vofs + 16, a
|
|
2863
|
+
i = ref[1]
|
|
2864
|
+
blk = self._read_block(dir_no, i, block_size)
|
|
2865
|
+
if blk[:4] != b'INDX' or not _apply_fixups(blk):
|
|
2866
|
+
raise NtfsError(errno.EIO, f'index block {i} unreadable')
|
|
2867
|
+
return blk, 24, None
|
|
2868
|
+
|
|
2869
|
+
def _node_flush(self, dir_no, ref, buf, block_size):
|
|
2870
|
+
if ref[0] == 'root':
|
|
2871
|
+
self.write_record(dir_no, buf)
|
|
2872
|
+
else:
|
|
2873
|
+
self._write_block(dir_no, ref[1], block_size, buf)
|
|
2874
|
+
|
|
2875
|
+
def _pack_or_grow(self, dir_no, ref, buf, hdr_off, root_attr_off,
|
|
2876
|
+
reals, end, block_size) -> bool:
|
|
2877
|
+
'''Pack and flush; grow a resident root if it overflows. Returns True if
|
|
2878
|
+
it fit (done), False for a block that must split. Root that can't
|
|
2879
|
+
grow → ENOTSUP (small→large / root split not implemented here).'''
|
|
2880
|
+
if self._pack_node(buf, hdr_off, reals, end):
|
|
2881
|
+
self._node_flush(dir_no, ref, buf, block_size)
|
|
2882
|
+
return True
|
|
2883
|
+
if ref[0] != 'root':
|
|
2884
|
+
return False
|
|
2885
|
+
eo = struct.unpack_from('<I', buf, hdr_off)[0]
|
|
2886
|
+
asz = struct.unpack_from('<I', buf, hdr_off + 8)[0]
|
|
2887
|
+
need = eo + len(b''.join(reals) + end)
|
|
2888
|
+
delta = (need - asz + 7) & ~7
|
|
2889
|
+
self._resident_grow_root(buf, root_attr_off, delta) # raises ENOTSUP if full
|
|
2890
|
+
if not self._pack_node(buf, hdr_off, reals, end):
|
|
2891
|
+
raise NtfsError(errno.ENOTSUP, 'root would need a height increase '
|
|
2892
|
+
'(small→large / root split) here — not implemented')
|
|
2893
|
+
self._node_flush(dir_no, ref, buf, block_size)
|
|
2894
|
+
return True
|
|
2895
|
+
|
|
2896
|
+
@staticmethod
|
|
2897
|
+
def _would_pack(hdr_off_asz_eo, reals, end) -> bool:
|
|
2898
|
+
asz, eo = hdr_off_asz_eo
|
|
2899
|
+
return eo + len(b''.join(reals) + end) <= asz
|
|
2900
|
+
|
|
2901
|
+
def insert_index_entry(self, dir_no: int, mref: int, fn_bytes: bytes) -> bool:
|
|
2902
|
+
'''Insert a $FILE_NAME leaf entry into a directory's $I30 in collation
|
|
2903
|
+
order. A full leaf splits and promotes its median to the parent
|
|
2904
|
+
(which grows if it is the resident root). Plan-then-commit and
|
|
2905
|
+
bounded: the whole operation is verified feasible BEFORE any write,
|
|
2906
|
+
and a split that would cascade past the immediate parent, need a root
|
|
2907
|
+
height increase, or overflow the $INDEX_ALLOCATION attribute record
|
|
2908
|
+
raises ENOTSUP with nothing written. EEXIST on a
|
|
2909
|
+
duplicate name.'''
|
|
2910
|
+
rec = self._load_record(dir_no)
|
|
2911
|
+
if rec is None:
|
|
2912
|
+
raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
|
|
2913
|
+
loc = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
|
|
2914
|
+
if not loc:
|
|
2915
|
+
raise NtfsError(errno.ENOENT, f'record {dir_no} has no $I30 index root')
|
|
2916
|
+
a, _ln = loc
|
|
2917
|
+
vofs = struct.unpack_from('<H', rec, a + 20)[0]
|
|
2918
|
+
block_size = struct.unpack_from('<I', rec, a + vofs + 8)[0]
|
|
2919
|
+
vpb = max(block_size // self._cluster_size, 1)
|
|
2920
|
+
new_key = self._ix.sort_key(fn_bytes) # collation key of the new entry
|
|
2921
|
+
entry = self._ix.build_leaf(mref, fn_bytes)
|
|
2922
|
+
|
|
2923
|
+
# -- descend, recording the path root→leaf --
|
|
2924
|
+
path, ref = [], ('root', dir_no)
|
|
2925
|
+
while True:
|
|
2926
|
+
buf, hdr_off, rao = self._node_buf(dir_no, ref, block_size)
|
|
2927
|
+
reals, end = self._decode_node(buf, hdr_off)
|
|
2928
|
+
idx, descend = None, None
|
|
2929
|
+
for k, e in enumerate(reals):
|
|
2930
|
+
ek = self._ix.entry_sort_key(e)
|
|
2931
|
+
if new_key == ek:
|
|
2932
|
+
raise NtfsError(errno.EEXIST, 'name already present in index')
|
|
2933
|
+
if new_key < ek:
|
|
2934
|
+
idx = k
|
|
2935
|
+
if struct.unpack_from('<H', e, 12)[0] & 1:
|
|
2936
|
+
descend = self._entry_subnode(e)
|
|
2937
|
+
break
|
|
2938
|
+
if idx is None:
|
|
2939
|
+
idx = len(reals)
|
|
2940
|
+
if struct.unpack_from('<H', end, 12)[0] & 1:
|
|
2941
|
+
descend = self._entry_subnode(end)
|
|
2942
|
+
asz = struct.unpack_from('<I', buf, hdr_off + 8)[0]
|
|
2943
|
+
eo = struct.unpack_from('<I', buf, hdr_off)[0]
|
|
2944
|
+
path.append({'ref': ref, 'buf': buf, 'hdr': hdr_off, 'rao': rao,
|
|
2945
|
+
'reals': reals, 'end': end, 'idx': idx, 'cap': (asz, eo)})
|
|
2946
|
+
if descend is None:
|
|
2947
|
+
break
|
|
2948
|
+
ref = ('block', descend // vpb)
|
|
2949
|
+
|
|
2950
|
+
leaf = path[-1]
|
|
2951
|
+
# small (root-only) index whose root can't hold the new entry even
|
|
2952
|
+
# grown → convert to a large index, then retry into the block tree
|
|
2953
|
+
if leaf['ref'][0] == 'root':
|
|
2954
|
+
body_len = len(b''.join(leaf['reals']) + [leaf['end']][0]) \
|
|
2955
|
+
+ len(entry)
|
|
2956
|
+
if not self._root_body_room(leaf['buf'], leaf['rao'], body_len):
|
|
2957
|
+
self._small_to_large(dir_no, block_size, vpb)
|
|
2958
|
+
return self.insert_index_entry(dir_no, mref, fn_bytes)
|
|
2959
|
+
leaf['reals'].insert(leaf['idx'], entry)
|
|
2960
|
+
# fast path: fits in the leaf (grow the root if that is the leaf)
|
|
2961
|
+
if self._pack_or_grow(dir_no, leaf['ref'], leaf['buf'], leaf['hdr'],
|
|
2962
|
+
leaf['rao'], leaf['reals'], leaf['end'], block_size):
|
|
2963
|
+
return True
|
|
2964
|
+
|
|
2965
|
+
# leaf overflow → multi-level atomic cascade split. Plan the whole
|
|
2966
|
+
# cascade bottom-up, allocating one block per split; if any allocation
|
|
2967
|
+
# fails (e.g. $INDEX_ALLOCATION mapping-pairs overflow) free them all and
|
|
2968
|
+
# refuse — nothing else is written, so the refusal is atomic. Only once
|
|
2969
|
+
# every block is reserved do we commit the node writes (which can't fail).
|
|
2970
|
+
if leaf['ref'][0] != 'block':
|
|
2971
|
+
raise NtfsError(errno.EIO, 'root leaf overflow after conversion check')
|
|
2972
|
+
allocated, writes, carry = [], [], None
|
|
2973
|
+
i = len(path) - 1
|
|
2974
|
+
try:
|
|
2975
|
+
while i >= 0:
|
|
2976
|
+
node = path[i]
|
|
2977
|
+
if carry is not None:
|
|
2978
|
+
self._insert_median_inplace(node, *carry)
|
|
2979
|
+
carry = None
|
|
2980
|
+
if node['ref'][0] == 'root': # root: grow or heighten
|
|
2981
|
+
fresh = self._load_record(dir_no)
|
|
2982
|
+
if fresh is None:
|
|
2983
|
+
raise NtfsError(errno.EIO,
|
|
2984
|
+
f'directory record {dir_no} unreadable')
|
|
2985
|
+
ra_f, _l = self._base_attr(fresh, AT_INDEX_ROOT, self._ix.name)
|
|
2986
|
+
body_len = len(b''.join(node['reals']) + node['end'])
|
|
2987
|
+
# keep the resident root small (a third of the record) so the
|
|
2988
|
+
# base has room for $INDEX_ALLOCATION + $BITMAP — heighten to
|
|
2989
|
+
# a tiny END→child pointer early rather than growing a fat 2-level
|
|
2990
|
+
# root that starves the base
|
|
2991
|
+
if body_len <= self._rec_size // 3 \
|
|
2992
|
+
and self._root_body_room(fresh, ra_f, body_len):
|
|
2993
|
+
writes.append(('rootfit', node))
|
|
2994
|
+
else:
|
|
2995
|
+
b_i, b_vcn = self._alloc_index_block(dir_no, block_size, vpb)
|
|
2996
|
+
allocated.append(b_i)
|
|
2997
|
+
writes.append(('rootheighten', node, b_vcn))
|
|
2998
|
+
break
|
|
2999
|
+
if self._would_pack(node['cap'], node['reals'], node['end']):
|
|
3000
|
+
writes.append(('blockfit', node))
|
|
3001
|
+
break
|
|
3002
|
+
end_is_node = bool(struct.unpack_from('<H', node['end'], 12)[0] & 1)
|
|
3003
|
+
mid = len(node['reals']) // 2
|
|
3004
|
+
median_e = node['reals'][mid]
|
|
3005
|
+
left, right = node['reals'][:mid], node['reals'][mid + 1:]
|
|
3006
|
+
if end_is_node:
|
|
3007
|
+
left_end = self._end_node(self._entry_subnode(median_e))
|
|
3008
|
+
right_end = node['end']
|
|
3009
|
+
else:
|
|
3010
|
+
left_end = right_end = self._leaf_end()
|
|
3011
|
+
my_vcn = node['ref'][1] * vpb
|
|
3012
|
+
new_i, new_vcn = self._alloc_index_block(dir_no, block_size, vpb)
|
|
3013
|
+
allocated.append(new_i)
|
|
3014
|
+
writes.append(('split', node, left, left_end, right, right_end, new_vcn))
|
|
3015
|
+
carry = (self._make_node_entry(median_e, my_vcn), my_vcn, new_vcn)
|
|
3016
|
+
i -= 1
|
|
3017
|
+
else:
|
|
3018
|
+
raise NtfsError(errno.EIO, 'cascade ran off the top of the tree')
|
|
3019
|
+
except NtfsError:
|
|
3020
|
+
for bi in allocated:
|
|
3021
|
+
self._free_index_block(dir_no, bi)
|
|
3022
|
+
raise
|
|
3023
|
+
|
|
3024
|
+
for w in writes: # commit — cannot fail
|
|
3025
|
+
if w[0] == 'split':
|
|
3026
|
+
_, node, left, left_end, right, right_end, new_vcn = w
|
|
3027
|
+
if not self._pack_node(node['buf'], node['hdr'], left, left_end):
|
|
3028
|
+
raise NtfsError(errno.EIO, 'left half overflow (unexpected)')
|
|
3029
|
+
self._write_block(dir_no, node['ref'][1], block_size, node['buf'])
|
|
3030
|
+
nb = self._new_indx_block(new_vcn, block_size)
|
|
3031
|
+
if not self._pack_node(nb, 24, right, right_end):
|
|
3032
|
+
raise NtfsError(errno.EIO, 'upper half overflow (unexpected)')
|
|
3033
|
+
self._write_block(dir_no, new_vcn // vpb, block_size, nb)
|
|
3034
|
+
elif w[0] == 'blockfit':
|
|
3035
|
+
node = w[1]
|
|
3036
|
+
self._pack_node(node['buf'], node['hdr'], node['reals'], node['end'])
|
|
3037
|
+
self._write_block(dir_no, node['ref'][1], block_size, node['buf'])
|
|
3038
|
+
elif w[0] == 'rootfit':
|
|
3039
|
+
node = w[1]
|
|
3040
|
+
self._write_root_node(dir_no, node['reals'], node['end'], small=False)
|
|
3041
|
+
elif w[0] == 'rootheighten':
|
|
3042
|
+
_, node, b_vcn = w
|
|
3043
|
+
bblk = self._new_indx_block(b_vcn, block_size)
|
|
3044
|
+
self._pack_node(bblk, 24, node['reals'], node['end'])
|
|
3045
|
+
self._write_block(dir_no, b_vcn // vpb, block_size, bblk)
|
|
3046
|
+
self._write_root_node(dir_no, [], self._end_node(b_vcn), small=False)
|
|
3047
|
+
return True
|
|
3048
|
+
|
|
3049
|
+
def _insert_median_inplace(self, node, median, left_vcn, right_vcn) -> None:
|
|
3050
|
+
'''Absorb a child's promoted median: find the pointer to the child that
|
|
3051
|
+
split (subnode == left_vcn), retarget it to the new right block, and
|
|
3052
|
+
insert the median (→left, the half that stayed) just before it.'''
|
|
3053
|
+
reals, end = node['reals'], node['end']
|
|
3054
|
+
for k, e in enumerate(reals):
|
|
3055
|
+
if struct.unpack_from('<H', e, 12)[0] & 1 \
|
|
3056
|
+
and self._entry_subnode(e) == left_vcn:
|
|
3057
|
+
reals[k] = self._set_subnode(e, right_vcn)
|
|
3058
|
+
reals.insert(k, median)
|
|
3059
|
+
return
|
|
3060
|
+
if struct.unpack_from('<H', end, 12)[0] & 1 \
|
|
3061
|
+
and self._entry_subnode(end) == left_vcn:
|
|
3062
|
+
node['end'] = self._set_subnode(end, right_vcn)
|
|
3063
|
+
reals.append(median)
|
|
3064
|
+
return
|
|
3065
|
+
raise NtfsError(errno.EIO, 'cascade: parent pointer to child not found')
|
|
3066
|
+
|
|
3067
|
+
def _free_index_block(self, dir_no, block_index) -> None:
|
|
3068
|
+
'''Clear an INDX block's $BITMAP bit (the block stays allocated in
|
|
3069
|
+
$INDEX_ALLOCATION — a benign leak a bitmap audit reclaims).'''
|
|
3070
|
+
bm = self._read_i30_bitmap(dir_no)
|
|
3071
|
+
bm[block_index >> 3] &= ~(1 << (block_index & 7))
|
|
3072
|
+
self._write_i30_bitmap(dir_no, bm)
|
|
3073
|
+
|
|
3074
|
+
# -- phase 4c: bulk-load rebuild (torn-index recovery from the MFT census) --
|
|
3075
|
+
|
|
3076
|
+
def _leaf_capacity(self, block_size: int) -> int:
|
|
3077
|
+
entries_off = ((40 + 2 * (1 + block_size // 512) + 7) & ~7) - 24
|
|
3078
|
+
return (block_size - 24) - entries_off # bytes for entries incl END
|
|
3079
|
+
|
|
3080
|
+
def _root_body_room(self, rec, root_attr_off: int, body_len: int) -> bool:
|
|
3081
|
+
vofs = struct.unpack_from('<H', rec, root_attr_off + 0x14)[0]
|
|
3082
|
+
cur_alen = struct.unpack_from('<I', rec, root_attr_off + 4)[0]
|
|
3083
|
+
in_use = struct.unpack_from('<I', rec, 0x18)[0]
|
|
3084
|
+
need_vlen = 16 + 16 + body_len # prefix + INDEX_HEADER + body
|
|
3085
|
+
need_alen = (vofs + need_vlen + 7) & ~7
|
|
3086
|
+
return need_alen - cur_alen <= len(rec) - in_use
|
|
3087
|
+
|
|
3088
|
+
def _write_root_node(self, dir_no, body, end, small: bool) -> None:
|
|
3089
|
+
rec = self._load_record(dir_no)
|
|
3090
|
+
if rec is None:
|
|
3091
|
+
raise NtfsError(errno.EIO, f'directory record {dir_no} unreadable')
|
|
3092
|
+
ra, _l = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
|
|
3093
|
+
vofs = struct.unpack_from('<H', rec, ra + 0x14)[0]
|
|
3094
|
+
cur_vlen = struct.unpack_from('<I', rec, ra + 0x10)[0]
|
|
3095
|
+
body_bytes = b''.join(body) + end
|
|
3096
|
+
need_vlen = 16 + 16 + len(body_bytes)
|
|
3097
|
+
if need_vlen != cur_vlen:
|
|
3098
|
+
# keep the attribute exactly-sized either way: Windows writes
|
|
3099
|
+
# tight roots, and chkdsk flags slack as an index error
|
|
3100
|
+
self._resident_grow_root(rec, ra, ((need_vlen + 7) & ~7) - cur_vlen)
|
|
3101
|
+
vofs = struct.unpack_from('<H', rec, ra + 0x14)[0]
|
|
3102
|
+
cur_vlen = struct.unpack_from('<I', rec, ra + 0x10)[0]
|
|
3103
|
+
val = ra + vofs # INDEX_ROOT prefix (16) kept
|
|
3104
|
+
struct.pack_into('<III', rec, val + 16, 16, 16 + len(body_bytes), cur_vlen - 16)
|
|
3105
|
+
rec[val + 16 + 12] = 0 if small else 1 # INDEX_HEADER.flags
|
|
3106
|
+
rec[val + 16 + 13:val + 16 + 16] = b'\x00\x00\x00'
|
|
3107
|
+
rec[val + 32:val + 32 + len(body_bytes)] = body_bytes
|
|
3108
|
+
for i in range(val + 32 + len(body_bytes), val + cur_vlen):
|
|
3109
|
+
rec[i] = 0
|
|
3110
|
+
self.write_record(dir_no, rec)
|
|
3111
|
+
|
|
3112
|
+
def _ensure_index_blocks(self, dir_no, block_size, vpb, nblocks) -> None:
|
|
3113
|
+
alloc = self._attr_value(dir_no, AT_INDEX_ALLOCATION, self._ix.name)
|
|
3114
|
+
have = len(alloc) // block_size
|
|
3115
|
+
while have < nblocks:
|
|
3116
|
+
self._grow_index_alloc(dir_no, block_size, vpb,
|
|
3117
|
+
self._read_i30_bitmap(dir_no), have)
|
|
3118
|
+
have += 1
|
|
3119
|
+
|
|
3120
|
+
def _set_i30_bitmap_used(self, dir_no, nblocks) -> None:
|
|
3121
|
+
bm = self._read_i30_bitmap(dir_no)
|
|
3122
|
+
for i in range(len(bm) * 8):
|
|
3123
|
+
if i < nblocks:
|
|
3124
|
+
bm[i >> 3] |= 1 << (i & 7)
|
|
3125
|
+
else:
|
|
3126
|
+
bm[i >> 3] &= ~(1 << (i & 7))
|
|
3127
|
+
self._write_i30_bitmap(dir_no, bm)
|
|
3128
|
+
|
|
3129
|
+
def rebuild_index(self, dir_no: int) -> dict:
|
|
3130
|
+
'''Rebuild a directory's $I30 from the MFT census (chkdsk stage-2 rebuild,
|
|
3131
|
+
native): build sorted $FILE_NAME leaf entries and bulk-load them.'''
|
|
3132
|
+
census = self.children_from_mft(dir_no)['children']
|
|
3133
|
+
items = []
|
|
3134
|
+
for c in census:
|
|
3135
|
+
e = self._ix.build_leaf(c['record'] | (c['seq'] << 48), c['fn_bytes'])
|
|
3136
|
+
items.append((self._ix.entry_sort_key(e), e))
|
|
3137
|
+
items.sort(key=lambda t: t[0])
|
|
3138
|
+
return self._bulk_write_index(dir_no, [e for _, e in items])
|
|
3139
|
+
|
|
3140
|
+
def _bulk_write_index(self, owner: int, entries: list) -> dict:
|
|
3141
|
+
'''Write `entries` (already in self._ix collation order) as the index
|
|
3142
|
+
named self._ix.name on record `owner`: a resident root if they fit,
|
|
3143
|
+
else a bottom-up bulk-loaded tree — one INDX block, or several with
|
|
3144
|
+
the root a single END→internal pointer and separators one level down.
|
|
3145
|
+
Shared by the $I30 directory rebuild and the $SII/$SDH $Secure rebuild;
|
|
3146
|
+
schema-generic — the separator format follows self._ix.'''
|
|
3147
|
+
rec = self._load_record(owner)
|
|
3148
|
+
if rec is None:
|
|
3149
|
+
raise NtfsError(errno.EIO, f'record {owner} unreadable')
|
|
3150
|
+
loc = self._base_attr(rec, AT_INDEX_ROOT, self._ix.name)
|
|
3151
|
+
if not loc:
|
|
3152
|
+
raise NtfsError(errno.ENOENT, f'record {owner} has no index root')
|
|
3153
|
+
ra, _l = loc
|
|
3154
|
+
rvofs = ra + struct.unpack_from('<H', rec, ra + 0x14)[0]
|
|
3155
|
+
block_size = struct.unpack_from('<I', rec, rvofs + 8)[0]
|
|
3156
|
+
vpb = max(block_size // self._cluster_size, 1)
|
|
3157
|
+
has_ia = self._base_attr(rec, AT_INDEX_ALLOCATION, self._ix.name) is not None
|
|
3158
|
+
leaf_end = self._leaf_end()
|
|
3159
|
+
|
|
3160
|
+
# small enough for the resident root, and no allocation to reconcile?
|
|
3161
|
+
if not has_ia:
|
|
3162
|
+
if self._root_body_room(rec, ra, len(b''.join(entries) + leaf_end)):
|
|
3163
|
+
self._write_root_node(owner, entries, leaf_end, small=True)
|
|
3164
|
+
return {'entries': len(entries), 'blocks': 0}
|
|
3165
|
+
# needs a large index but has none — create $INDEX_ALLOCATION +
|
|
3166
|
+
# $BITMAP, then fall through (the large build overwrites block 0)
|
|
3167
|
+
self._small_to_large(owner, block_size, vpb)
|
|
3168
|
+
if getattr(self, '_ia_cache', None):
|
|
3169
|
+
self._ia_cache.pop(owner, None)
|
|
3170
|
+
|
|
3171
|
+
# large: pack leaves. Root stays a single END→child pointer; separators
|
|
3172
|
+
# live in an internal block one level down (keeps the resident root tiny).
|
|
3173
|
+
cap = self._leaf_capacity(block_size)
|
|
3174
|
+
leaves, cur, cur_len = [], [], 0
|
|
3175
|
+
for e in entries:
|
|
3176
|
+
if cur and cur_len + len(e) + 16 > cap: # +16 keeps END room
|
|
3177
|
+
leaves.append(cur)
|
|
3178
|
+
cur, cur_len = [], 0
|
|
3179
|
+
cur.append(e)
|
|
3180
|
+
cur_len += len(e)
|
|
3181
|
+
leaves.append(cur)
|
|
3182
|
+
nleaves = len(leaves)
|
|
3183
|
+
|
|
3184
|
+
if nleaves == 1: # one leaf block; root → it
|
|
3185
|
+
self._ensure_index_blocks(owner, block_size, vpb, 1)
|
|
3186
|
+
blk = self._new_indx_block(0, block_size)
|
|
3187
|
+
self._pack_node(blk, 24, leaves[0], leaf_end)
|
|
3188
|
+
self._write_block(owner, 0, block_size, blk)
|
|
3189
|
+
self._set_i30_bitmap_used(owner, 1)
|
|
3190
|
+
self._write_root_node(owner, [], self._end_node(0), small=False)
|
|
3191
|
+
return {'entries': len(entries), 'blocks': 1}
|
|
3192
|
+
|
|
3193
|
+
# build the tree bottom-up to arbitrary depth (root stays a single
|
|
3194
|
+
# END→child pointer; leaves are blocks 0..nleaves-1, internal levels
|
|
3195
|
+
# take the blocks above them).
|
|
3196
|
+
leaf_nodes = [{'vcn': i * vpb, 'entries': list(leaves[i]), 'end_vcn': None}
|
|
3197
|
+
for i in range(nleaves)]
|
|
3198
|
+
planned = [] # (block_index, body, end)
|
|
3199
|
+
top_vcn, total = self._build_tree(leaf_nodes, block_size, vpb,
|
|
3200
|
+
nleaves, planned)
|
|
3201
|
+
self._ensure_index_blocks(owner, block_size, vpb, total)
|
|
3202
|
+
for bi, body, end in planned:
|
|
3203
|
+
blk = self._new_indx_block(bi * vpb, block_size)
|
|
3204
|
+
if not self._pack_node(blk, 24, body, end):
|
|
3205
|
+
raise NtfsError(errno.EIO, f'node {bi} body overflow (unexpected)')
|
|
3206
|
+
self._write_block(owner, bi, block_size, blk)
|
|
3207
|
+
self._set_i30_bitmap_used(owner, total)
|
|
3208
|
+
self._write_root_node(owner, [], self._end_node(top_vcn), small=False)
|
|
3209
|
+
return {'entries': len(entries), 'blocks': total}
|
|
3210
|
+
|
|
3211
|
+
def _build_tree(self, nodes, block_size, vpb, next_bi, planned):
|
|
3212
|
+
'''Bottom-up bulk load. `nodes` is an ordered list of
|
|
3213
|
+
{vcn, entries, end_vcn} (end_vcn None marks a leaf). Records each
|
|
3214
|
+
node into `planned` and returns (top_vcn, total_block_count). The
|
|
3215
|
+
parent level is built by promoting the first entry of every non-first
|
|
3216
|
+
sibling as a separator (the standard NTFS B+ convention), recursing
|
|
3217
|
+
until one node remains.'''
|
|
3218
|
+
if len(nodes) == 1:
|
|
3219
|
+
n = nodes[0]
|
|
3220
|
+
end = self._leaf_end() if n['end_vcn'] is None else self._end_node(n['end_vcn'])
|
|
3221
|
+
planned.append((n['vcn'] // vpb, n['entries'], end))
|
|
3222
|
+
return n['vcn'], next_bi
|
|
3223
|
+
|
|
3224
|
+
seps = [] # (first_entry, left_child_vcn)
|
|
3225
|
+
for i in range(1, len(nodes)):
|
|
3226
|
+
seps.append((nodes[i]['entries'].pop(0), nodes[i - 1]['vcn']))
|
|
3227
|
+
final_child = nodes[-1]['vcn']
|
|
3228
|
+
for n in nodes: # this level is now final
|
|
3229
|
+
end = self._leaf_end() if n['end_vcn'] is None else self._end_node(n['end_vcn'])
|
|
3230
|
+
planned.append((n['vcn'] // vpb, n['entries'], end))
|
|
3231
|
+
|
|
3232
|
+
cap = self._leaf_capacity(block_size) - 24 # reserve the node END entry
|
|
3233
|
+
groups, cur, cur_len = [], [], 0
|
|
3234
|
+
for e, child in seps:
|
|
3235
|
+
ne = self._make_node_entry(e, child)
|
|
3236
|
+
if cur and cur_len + len(ne) > cap:
|
|
3237
|
+
groups.append(cur)
|
|
3238
|
+
cur, cur_len = [], 0
|
|
3239
|
+
cur.append((ne, child))
|
|
3240
|
+
cur_len += len(ne)
|
|
3241
|
+
groups.append(cur)
|
|
3242
|
+
|
|
3243
|
+
parents, bi = [], next_bi
|
|
3244
|
+
for j, group in enumerate(groups):
|
|
3245
|
+
end_vcn = final_child if j == len(groups) - 1 else groups[j + 1][0][1]
|
|
3246
|
+
parents.append({'vcn': bi * vpb, 'end_vcn': end_vcn,
|
|
3247
|
+
'entries': [ne for ne, _c in group]})
|
|
3248
|
+
bi += 1
|
|
3249
|
+
return self._build_tree(parents, block_size, vpb, bi, planned)
|
|
3250
|
+
|
|
3251
|
+
def rebuild_dir(self, path: str) -> dict:
|
|
3252
|
+
'''Adapter so full_fix's rebuild step works on the native engine —
|
|
3253
|
+
returns the {added, duplicates, planned} shape full_fix expects.'''
|
|
3254
|
+
r = self.rebuild_index(self._resolve(path))
|
|
3255
|
+
return {'added': r['entries'], 'duplicates': 0, 'planned': r['entries']}
|
|
3256
|
+
|
|
3257
|
+
def reconnect_lost(self, lost: dict) -> int:
|
|
3258
|
+
'''Lost-file reconnection: re-add each $FILE_NAME leaf entry into
|
|
3259
|
+
the live parent's index.'''
|
|
3260
|
+
self._invalidate_mft_cache()
|
|
3261
|
+
mref = lost['record'] | (lost['seq'] << 48)
|
|
3262
|
+
added = 0
|
|
3263
|
+
for fn in lost['fn_list']:
|
|
3264
|
+
try:
|
|
3265
|
+
self.insert_index_entry(lost['parent'], mref, fn)
|
|
3266
|
+
added += 1
|
|
3267
|
+
except NtfsError as exc:
|
|
3268
|
+
if exc.errno != errno.EEXIST:
|
|
3269
|
+
raise
|
|
3270
|
+
return added
|
|
3271
|
+
|
|
3272
|
+
# -- $LogFile reset (the ntfsfix/chkdsk approach — a reset, NOT a replay) --
|
|
3273
|
+
|
|
3274
|
+
def reset_logfile(self) -> int:
|
|
3275
|
+
'''Empty $LogFile by filling it with 0xff, so the driver and Windows
|
|
3276
|
+
treat the journal as clean and replay nothing. This DISCARDS any
|
|
3277
|
+
pending crash-journal transactions — it is not a replay of them;
|
|
3278
|
+
the structural checks reconcile the on-disk state instead, exactly
|
|
3279
|
+
as chkdsk does (which also resets, not replays, the log).'''
|
|
3280
|
+
size, runs = self._stream_runs(FILE_LOGFILE, AT_DATA, None)
|
|
3281
|
+
if not runs:
|
|
3282
|
+
raise NtfsError(errno.ENOENT, '$LogFile has no non-resident $DATA')
|
|
3283
|
+
chunk = b'\xff' * (1 << 20)
|
|
3284
|
+
off = 0
|
|
3285
|
+
while off < size:
|
|
3286
|
+
self._runs_write(runs, off, chunk[:min(len(chunk), size - off)])
|
|
3287
|
+
off += len(chunk)
|
|
3288
|
+
return size
|
|
3289
|
+
|
|
3290
|
+
def clear_dirty(self) -> None:
|
|
3291
|
+
'''Clear the volume dirty flag — only valid after checks pass clean.'''
|
|
3292
|
+
self._was_dirty = False # so close() does not re-clear or preserve
|
|
3293
|
+
self._set_dirty(False)
|
|
3294
|
+
|
|
3295
|
+
|
|
3296
|
+
# -- phase 5: purge composition (unlink + free record + free clusters) --
|
|
3297
|
+
|
|
3298
|
+
def _extension_records(self, rec_no: int) -> set:
|
|
3299
|
+
'''Extension record numbers a base record's $ATTRIBUTE_LIST references
|
|
3300
|
+
(empty if the file lives in a single record) '''
|
|
3301
|
+
|
|
3302
|
+
rec = self._load_record(rec_no)
|
|
3303
|
+
if rec is None:
|
|
3304
|
+
return set()
|
|
3305
|
+
frozen = bytes(rec)
|
|
3306
|
+
exts, nc = set(), self.nr_clusters()
|
|
3307
|
+
for a, t, ln in _attrs(frozen):
|
|
3308
|
+
if t != AT_ATTRIBUTE_LIST:
|
|
3309
|
+
continue
|
|
3310
|
+
if frozen[a + 8] == 0:
|
|
3311
|
+
vo = struct.unpack_from('<H', frozen, a + 20)[0]
|
|
3312
|
+
vl = struct.unpack_from('<I', frozen, a + 16)[0]
|
|
3313
|
+
listing = frozen[a + vo:a + vo + vl]
|
|
3314
|
+
else:
|
|
3315
|
+
_p, _ph, r = _check_mapping_pairs(frozen, a, ln, nc)
|
|
3316
|
+
size = struct.unpack_from('<q', frozen, a + 48)[0]
|
|
3317
|
+
listing = self._runs_read(r, 0, size)
|
|
3318
|
+
pos = 0
|
|
3319
|
+
while pos + 26 <= len(listing):
|
|
3320
|
+
e_len = struct.unpack_from('<H', listing, pos + 4)[0]
|
|
3321
|
+
if e_len < 26:
|
|
3322
|
+
break
|
|
3323
|
+
mref = struct.unpack_from('<Q', listing, pos + 16)[0] & MREF_MASK
|
|
3324
|
+
if mref != rec_no:
|
|
3325
|
+
exts.add(mref)
|
|
3326
|
+
pos += e_len
|
|
3327
|
+
return exts
|
|
3328
|
+
|
|
3329
|
+
|
|
3330
|
+
def _record_alloc_runs(self, rec_no: int):
|
|
3331
|
+
'''(lcn, run_len) clusters owned by every non-resident attribute of the
|
|
3332
|
+
file — spanning extension records when an $ATTRIBUTE_LIST is present
|
|
3333
|
+
(_record_attrs follows the list) '''
|
|
3334
|
+
|
|
3335
|
+
runs, nc = [], self.nr_clusters()
|
|
3336
|
+
for frozen, a, length in self._record_attrs(rec_no):
|
|
3337
|
+
if frozen[a + 8] == 1: # non-resident
|
|
3338
|
+
_p, _ph, r = _check_mapping_pairs(frozen, a, length, nc)
|
|
3339
|
+
runs += [(lcn, rlen) for lcn, rlen, _v in r]
|
|
3340
|
+
return runs
|
|
3341
|
+
|
|
3342
|
+
|
|
3343
|
+
def _free_file_record(self, rec_no: int) -> None:
|
|
3344
|
+
'''Clear MFT_RECORD_IN_USE, bump the sequence (so stale refs detect as
|
|
3345
|
+
stale), and clear the $MFT bitmap bit '''
|
|
3346
|
+
|
|
3347
|
+
rec = self._load_record(rec_no)
|
|
3348
|
+
if rec is None:
|
|
3349
|
+
raise NtfsError(errno.EIO, f'record {rec_no} unreadable')
|
|
3350
|
+
flags = struct.unpack_from('<H', rec, 22)[0]
|
|
3351
|
+
struct.pack_into('<H', rec, 22, flags & ~1)
|
|
3352
|
+
seq = struct.unpack_from('<H', rec, 16)[0]
|
|
3353
|
+
struct.pack_into('<H', rec, 16, (seq + 1) & 0xFFFF or 1)
|
|
3354
|
+
self.write_record(rec_no, rec)
|
|
3355
|
+
self.free_record(rec_no)
|
|
3356
|
+
|
|
3357
|
+
|
|
3358
|
+
def purge_orphan(self, path: str, name: str, really: bool) -> dict:
|
|
3359
|
+
''' Reclaim a true orphan natively. crash-safe order : unlink, clear the
|
|
3360
|
+
record's in-use flag, free clusters, so no live record ever references
|
|
3361
|
+
freed space '''
|
|
3362
|
+
|
|
3363
|
+
verdict = self.classify_dirent(path, name)
|
|
3364
|
+
if verdict['state'] != 'orphan':
|
|
3365
|
+
raise SystemExit(f'refusing purge: entry is {verdict["state"]!r} '
|
|
3366
|
+
f'— {verdict["action"]}')
|
|
3367
|
+
if not really:
|
|
3368
|
+
return verdict
|
|
3369
|
+
dir_no = self._resolve(path)
|
|
3370
|
+
rec_no = verdict['dirent_record']
|
|
3371
|
+
runs = self._record_alloc_runs(rec_no) # all clusters (base + extensions)
|
|
3372
|
+
exts = self._extension_records(rec_no) # extension records to free too
|
|
3373
|
+
# crash-safe order: unlink, free every record (in-use cleared) so no
|
|
3374
|
+
# live record references the clusters, then free the clusters
|
|
3375
|
+
self.remove_index_entry(dir_no, name)
|
|
3376
|
+
self._free_file_record(rec_no)
|
|
3377
|
+
for ext in sorted(exts):
|
|
3378
|
+
self._free_file_record(ext)
|
|
3379
|
+
if runs:
|
|
3380
|
+
self.free_clusters(runs)
|
|
3381
|
+
verdict['purged'] = True
|
|
3382
|
+
return verdict
|
|
3383
|
+
|
|
3384
|
+
# -- op 1: the torn-truncate terminator (native) --
|
|
3385
|
+
|
|
3386
|
+
def terminate_mapping_pairs(self, rec_no: int, mp_off: int) -> None:
|
|
3387
|
+
'''Zero the first mapping-pairs byte of a phantom-runs empty attribute.
|
|
3388
|
+
Whole-record sealed write, so fixup slots are no constraint — the
|
|
3389
|
+
sealer regenerates them.'''
|
|
3390
|
+
rec = self._load_record(rec_no)
|
|
3391
|
+
if rec is None:
|
|
3392
|
+
raise NtfsError(errno.EIO, f'record {rec_no} unreadable')
|
|
3393
|
+
if rec[mp_off] == 0:
|
|
3394
|
+
return # already terminated
|
|
3395
|
+
rec[mp_off] = 0
|
|
3396
|
+
self.write_record(rec_no, rec)
|
|
3397
|
+
|
|
3398
|
+
|
|
3399
|
+
def open_volume(device: str, readonly: bool = True):
|
|
3400
|
+
'''Open the volume: reads use RawVolume, writes use RawVolumeRW.'''
|
|
3401
|
+
return RawVolume(device) if readonly else RawVolumeRW(device)
|
|
3402
|
+
|
|
3403
|
+
|
|
3404
|
+
# ── full-volume pass: the chkdsk /f flow (the parts this tool can do) ────────
|
|
3405
|
+
|
|
3406
|
+
def run_surface(device: str, vol: RawVolume, direct: bool = False) -> dict:
|
|
3407
|
+
'''chkdsk /r stage 4, report-only: read every cluster of the volume
|
|
3408
|
+
(sequential 64 MiB chunks, bisecting failures down to clusters), then
|
|
3409
|
+
attribute unreadable clusters to their owning files.
|
|
3410
|
+
|
|
3411
|
+
direct=True opens with O_DIRECT (page-aligned mmap buffer + preadv) so
|
|
3412
|
+
a 500 GB scan does not evict the machine's page cache; falls back to
|
|
3413
|
+
buffered reads + posix_fadvise(DONTNEED) where O_DIRECT is refused.'''
|
|
3414
|
+
|
|
3415
|
+
end = vol.nr_clusters() * vol.cluster_size()
|
|
3416
|
+
fd = None
|
|
3417
|
+
if direct and hasattr(os, 'O_DIRECT'):
|
|
3418
|
+
try:
|
|
3419
|
+
fd = os.open(device, os.O_RDONLY | os.O_DIRECT | _O_BINARY)
|
|
3420
|
+
except OSError:
|
|
3421
|
+
print(' (O_DIRECT refused here — using buffered reads + fadvise)')
|
|
3422
|
+
if fd is None:
|
|
3423
|
+
direct = False
|
|
3424
|
+
fd = os.open(device, os.O_RDONLY | _O_BINARY)
|
|
3425
|
+
last = [0]
|
|
3426
|
+
|
|
3427
|
+
def progress(pos: int, total: int) -> None:
|
|
3428
|
+
if pos - last[0] >= (32 << 30):
|
|
3429
|
+
print(f' ... {pos >> 30} / {total >> 30} GiB read')
|
|
3430
|
+
last[0] = pos
|
|
3431
|
+
|
|
3432
|
+
if direct:
|
|
3433
|
+
import mmap
|
|
3434
|
+
dbuf = mmap.mmap(-1, 64 << 20) # page-aligned, as O_DIRECT requires
|
|
3435
|
+
|
|
3436
|
+
def pread(pos: int, count: int) -> bytes:
|
|
3437
|
+
got = os.preadv(fd, [memoryview(dbuf)[:count]], pos)
|
|
3438
|
+
return dbuf[:got]
|
|
3439
|
+
else:
|
|
3440
|
+
def pread(pos: int, count: int) -> bytes:
|
|
3441
|
+
data = _pread(fd, count, pos)
|
|
3442
|
+
if hasattr(os, 'posix_fadvise'): # don't evict the page cache
|
|
3443
|
+
os.posix_fadvise(fd, pos, count, os.POSIX_FADV_DONTNEED)
|
|
3444
|
+
return data
|
|
3445
|
+
|
|
3446
|
+
try:
|
|
3447
|
+
bad = _surface_scan(pread, end, vol.cluster_size(), progress)
|
|
3448
|
+
finally:
|
|
3449
|
+
os.close(fd)
|
|
3450
|
+
return {'bad': bad, 'bytes': end,
|
|
3451
|
+
'owners': vol.surface_owners(set(bad)) if bad else []}
|
|
3452
|
+
|
|
3453
|
+
|
|
3454
|
+
def _print_surface(scan: dict, indent: str = '') -> None:
|
|
3455
|
+
print(f'{indent}{scan["bytes"] >> 20} MiB read, '
|
|
3456
|
+
f'{len(scan["bad"])} unreadable cluster(s)')
|
|
3457
|
+
for own in scan['owners'][:10]:
|
|
3458
|
+
heads = ', '.join(str(c) for c in own['clusters'][:6])
|
|
3459
|
+
more = '...' if len(own['clusters']) > 6 else ''
|
|
3460
|
+
if own['record'] is None:
|
|
3461
|
+
print(f'{indent}! {len(own["clusters"])} bad cluster(s) in free space '
|
|
3462
|
+
f'({heads}{more})')
|
|
3463
|
+
else:
|
|
3464
|
+
print(f'{indent}! record {own["record"]} attr {own["attr"]:#x} '
|
|
3465
|
+
f'({own["names"]}): bad cluster(s) {heads}{more}')
|
|
3466
|
+
if scan['bad']:
|
|
3467
|
+
print(f'{indent}report-only: reallocate via a real chkdsk /r, or restore '
|
|
3468
|
+
'the affected files from backup')
|
|
3469
|
+
|
|
3470
|
+
|
|
3471
|
+
def full_fix(device: str, really: bool, surface: bool = False,
|
|
3472
|
+
direct: bool = False) -> int:
|
|
3473
|
+
'''The chkdsk /f flow, the parts this tool implements. Stage 1 verifies
|
|
3474
|
+
every MFT record and attribute runlist (auto-repairing torn empty-
|
|
3475
|
+
attribute truncates); stage 2 walks every directory index, repairing
|
|
3476
|
+
dangling entries and rebuilding torn indexes; stage 3 validates and
|
|
3477
|
+
repairs $Secure; stage 5 reconciles $Bitmap accounting; the USN journal
|
|
3478
|
+
is validated and reset. Stage 4 (surface scan) is report-only.'''
|
|
3479
|
+
|
|
3480
|
+
extra_issues = extra_fixed = extra_failed = 0
|
|
3481
|
+
with open_volume(device, readonly=not really) as vol:
|
|
3482
|
+
print('stage 1: examining MFT records and attribute runlists ...')
|
|
3483
|
+
survey = vol.mft_survey(want_used=True, want_security=True)
|
|
3484
|
+
health = survey
|
|
3485
|
+
print(f' {health["total"]} records, {health["in_use"]} in use '
|
|
3486
|
+
f'({health["dirs"]} directories), {health["torn"]} torn, '
|
|
3487
|
+
f'{len(health["runlist_problems"])} runlist problem(s)')
|
|
3488
|
+
for prob in health['runlist_problems']:
|
|
3489
|
+
extra_issues += 1
|
|
3490
|
+
print(f'! record {prob["record"]} attr {prob["attr_type"]:#x} '
|
|
3491
|
+
f'({prob["names"]}): {prob["problem"]}')
|
|
3492
|
+
if prob['fix_mp_off'] is None:
|
|
3493
|
+
print(' no safe auto-repair — chkdsk territory')
|
|
3494
|
+
if really:
|
|
3495
|
+
extra_failed += 1
|
|
3496
|
+
continue
|
|
3497
|
+
if not really:
|
|
3498
|
+
print(' (safe auto-repair available: terminate the phantom runlist)')
|
|
3499
|
+
continue
|
|
3500
|
+
try:
|
|
3501
|
+
vol.terminate_mapping_pairs(prob['record'], prob['fix_mp_off'])
|
|
3502
|
+
extra_fixed += 1
|
|
3503
|
+
print(' fixed (runlist terminated; any leaked clusters are '
|
|
3504
|
+
'reclaimed in stage 5)')
|
|
3505
|
+
except NtfsError as exc:
|
|
3506
|
+
extra_failed += 1
|
|
3507
|
+
print(f' FIX FAILED: {exc}')
|
|
3508
|
+
|
|
3509
|
+
# $MFT's own $BITMAP (bit i = record i allocated): a crash between the
|
|
3510
|
+
# bitmap write and the record write leaks set bits — chkdsk's "the
|
|
3511
|
+
# master file table's (MFT) BITMAP attribute is incorrect". Reconcile
|
|
3512
|
+
# against the records' surveyed in-use flags, the on-disk truth.
|
|
3513
|
+
print('mft record bitmap: verifying ...')
|
|
3514
|
+
mft_bmp = bytes(vol._read_whole_attr(0, AT_BITMAP, None, None) or b'')
|
|
3515
|
+
rec_used = survey['rec_used']
|
|
3516
|
+
nbits = min(survey['n_slots'], len(mft_bmp) * 8)
|
|
3517
|
+
mft_mism = [i for i in range(nbits)
|
|
3518
|
+
if bool(rec_used[i >> 3] & (1 << (i & 7)))
|
|
3519
|
+
!= bool(mft_bmp[i >> 3] & (1 << (i & 7)))]
|
|
3520
|
+
if len(mft_bmp) * 8 < survey['n_slots']:
|
|
3521
|
+
extra_issues += 1
|
|
3522
|
+
print(f'! $BITMAP covers {len(mft_bmp) * 8} of '
|
|
3523
|
+
f'{survey["n_slots"]} record slots — truncated (report-only)')
|
|
3524
|
+
if really:
|
|
3525
|
+
extra_failed += 1
|
|
3526
|
+
if mft_mism:
|
|
3527
|
+
extra_issues += 1
|
|
3528
|
+
print(f'! {len(mft_mism)} record(s) disagree with the bitmap '
|
|
3529
|
+
f'(first: {mft_mism[:8]})')
|
|
3530
|
+
if not really:
|
|
3531
|
+
print(' (repairable: --really rewrites the bitmap from the '
|
|
3532
|
+
'surveyed in-use flags)')
|
|
3533
|
+
else:
|
|
3534
|
+
n = vol.apply_mft_bitmap(rec_used, nbits)
|
|
3535
|
+
again = bytes(vol._read_whole_attr(0, AT_BITMAP, None, None)
|
|
3536
|
+
or b'')
|
|
3537
|
+
left = [i for i in range(nbits)
|
|
3538
|
+
if bool(rec_used[i >> 3] & (1 << (i & 7)))
|
|
3539
|
+
!= bool(again[i >> 3] & (1 << (i & 7)))]
|
|
3540
|
+
if left:
|
|
3541
|
+
extra_failed += 1
|
|
3542
|
+
print(' REWRITE VERIFY FAILED')
|
|
3543
|
+
else:
|
|
3544
|
+
extra_fixed += 1
|
|
3545
|
+
print(f' bitmap rewritten ({n} bit(s) corrected)')
|
|
3546
|
+
else:
|
|
3547
|
+
print(' consistent')
|
|
3548
|
+
|
|
3549
|
+
print('stage 2: examining directory indexes ...')
|
|
3550
|
+
walked = dangling = torn = fixed = failed = 0
|
|
3551
|
+
stack, seen = [('/', FILE_ROOT)], {FILE_ROOT}
|
|
3552
|
+
referenced: set[int] = set()
|
|
3553
|
+
while stack:
|
|
3554
|
+
path, dir_no = stack.pop()
|
|
3555
|
+
walked += 1
|
|
3556
|
+
if walked % 2000 == 0:
|
|
3557
|
+
print(f' ... {walked} directories walked, {len(stack)} queued')
|
|
3558
|
+
|
|
3559
|
+
probe = vol.probe_dir_no(dir_no)
|
|
3560
|
+
if not probe['readable']:
|
|
3561
|
+
torn += 1
|
|
3562
|
+
print(f'! torn index: {path!r} — {probe["error"]}')
|
|
3563
|
+
if not really:
|
|
3564
|
+
print(' (children not walked; --really will rebuild this index)')
|
|
3565
|
+
continue
|
|
3566
|
+
try:
|
|
3567
|
+
result = vol.rebuild_dir(path)
|
|
3568
|
+
print(f' rebuilt: {result["added"]} entries re-added')
|
|
3569
|
+
except (SystemExit, NtfsError) as exc:
|
|
3570
|
+
failed += 1
|
|
3571
|
+
print(f' REBUILD FAILED: {exc}')
|
|
3572
|
+
continue
|
|
3573
|
+
probe = vol.probe_dir_no(dir_no)
|
|
3574
|
+
if not probe['readable']:
|
|
3575
|
+
failed += 1
|
|
3576
|
+
print(f' VERIFY FAILED: still unreadable — {probe["error"]}')
|
|
3577
|
+
continue
|
|
3578
|
+
fixed += 1
|
|
3579
|
+
|
|
3580
|
+
for entry in probe['entries']:
|
|
3581
|
+
full = (path.rstrip('/')) + '/' + entry['name']
|
|
3582
|
+
if entry['status'] == 'ok':
|
|
3583
|
+
referenced.add(entry['record'])
|
|
3584
|
+
if entry.get('is_dir') and entry['record'] not in seen:
|
|
3585
|
+
seen.add(entry['record'])
|
|
3586
|
+
stack.append((full, entry['record']))
|
|
3587
|
+
continue
|
|
3588
|
+
dangling += 1
|
|
3589
|
+
try:
|
|
3590
|
+
verdict = vol.classify_dirent(path, entry['name'])
|
|
3591
|
+
except NtfsError as exc:
|
|
3592
|
+
if really and exc.errno == errno.ENOENT:
|
|
3593
|
+
# A purge of a sibling name (WIN32/DOS twin) of the same
|
|
3594
|
+
# file already removed this entry along with it.
|
|
3595
|
+
print(f'! {full!r}: entry already removed with its twin name')
|
|
3596
|
+
fixed += 1
|
|
3597
|
+
else:
|
|
3598
|
+
failed += 1
|
|
3599
|
+
print(f'! {full!r}: triage failed: {exc}')
|
|
3600
|
+
continue
|
|
3601
|
+
print(f'! {full!r}: {verdict["state"]} -> {verdict["action"]}')
|
|
3602
|
+
if not really or verdict['state'] == 'healthy':
|
|
3603
|
+
continue
|
|
3604
|
+
try:
|
|
3605
|
+
if verdict['state'] == 'orphan':
|
|
3606
|
+
vol.purge_orphan(path, entry['name'], really=True)
|
|
3607
|
+
else:
|
|
3608
|
+
vol.remove_dirent(path, entry['name'], really=True)
|
|
3609
|
+
fixed += 1
|
|
3610
|
+
print(' fixed')
|
|
3611
|
+
except (SystemExit, NtfsError) as exc:
|
|
3612
|
+
failed += 1
|
|
3613
|
+
print(f' FIX FAILED: {exc}')
|
|
3614
|
+
|
|
3615
|
+
print('lost files: records no index entry references ...')
|
|
3616
|
+
lost = vol.find_lost_files(referenced)
|
|
3617
|
+
print(f' {len(lost)} lost file(s)')
|
|
3618
|
+
for lf in lost:
|
|
3619
|
+
extra_issues += 1
|
|
3620
|
+
where = (f"live parent {lf['parent']}" if lf['parent_ok']
|
|
3621
|
+
else f"parent {lf['parent']} is dead/reused — not reconnectable")
|
|
3622
|
+
print(f"! lost file {lf['name']!r} (record {lf['record']}): {where}")
|
|
3623
|
+
if not lf['parent_ok']:
|
|
3624
|
+
if really:
|
|
3625
|
+
extra_failed += 1
|
|
3626
|
+
print(' chkdsk territory (found.000-style recovery not implemented)')
|
|
3627
|
+
continue
|
|
3628
|
+
if not really:
|
|
3629
|
+
print(' (repairable: --really re-adds it to its parent directory)')
|
|
3630
|
+
continue
|
|
3631
|
+
try:
|
|
3632
|
+
added = vol.reconnect_lost(lf)
|
|
3633
|
+
if added:
|
|
3634
|
+
extra_fixed += 1
|
|
3635
|
+
print(f' reconnected ({added} index entr'
|
|
3636
|
+
f'{"y" if added == 1 else "ies"} re-added)')
|
|
3637
|
+
else:
|
|
3638
|
+
extra_failed += 1
|
|
3639
|
+
print(' NOT reconnected: name already taken in the parent')
|
|
3640
|
+
except NtfsError as exc:
|
|
3641
|
+
extra_failed += 1
|
|
3642
|
+
print(f' RECONNECT FAILED: {exc}')
|
|
3643
|
+
|
|
3644
|
+
print('stage 3: verifying security descriptors ($Secure) ...')
|
|
3645
|
+
sec = vol.secure_check(survey=survey)
|
|
3646
|
+
for note in sec['notes']:
|
|
3647
|
+
print(f' note: {note}')
|
|
3648
|
+
if sec['present']:
|
|
3649
|
+
print(f' {sec["descriptors"]} descriptors, {sec["files_checked"]} files '
|
|
3650
|
+
f'checked: {len(sec["mirror_fixes"])} mirror mismatch(es), '
|
|
3651
|
+
f'{len(sec["index_problems"])} index problem(s), '
|
|
3652
|
+
f'{len(sec["problems"])} unrepairable, '
|
|
3653
|
+
f'{len(sec["ref_missing"])} dangling reference(s)')
|
|
3654
|
+
for prob in (sec['problems'] + sec['index_problems']
|
|
3655
|
+
+ sec['ref_missing'])[:10]:
|
|
3656
|
+
print(f'! {prob}')
|
|
3657
|
+
sec_issues = (bool(sec['mirror_fixes']) + bool(sec['index_problems'])
|
|
3658
|
+
+ bool(sec['problems']) + bool(sec['ref_missing']))
|
|
3659
|
+
extra_issues += sec_issues
|
|
3660
|
+
if sec_issues and really:
|
|
3661
|
+
if sec['mirror_fixes']:
|
|
3662
|
+
try:
|
|
3663
|
+
n = vol.secure_fix_mirrors(sec['mirror_fixes'])
|
|
3664
|
+
extra_fixed += 1
|
|
3665
|
+
print(f' {n} $SDS mirror pair(s) repaired from the good copy')
|
|
3666
|
+
except NtfsError as exc:
|
|
3667
|
+
extra_failed += 1
|
|
3668
|
+
print(f' MIRROR FIX FAILED: {exc}')
|
|
3669
|
+
if sec['index_problems']:
|
|
3670
|
+
try:
|
|
3671
|
+
result = vol.rebuild_secure_indexes(sec['sds_entries'])
|
|
3672
|
+
again = vol.secure_check()
|
|
3673
|
+
if again['index_problems']:
|
|
3674
|
+
extra_failed += 1
|
|
3675
|
+
print(' REBUILD VERIFY FAILED: '
|
|
3676
|
+
+ again['index_problems'][0])
|
|
3677
|
+
else:
|
|
3678
|
+
extra_fixed += 1
|
|
3679
|
+
print(f' $SII/$SDH rebuilt from $SDS '
|
|
3680
|
+
f'({result["added"]} descriptors)')
|
|
3681
|
+
except NtfsError as exc:
|
|
3682
|
+
extra_failed += 1
|
|
3683
|
+
print(f' REBUILD FAILED: {exc}')
|
|
3684
|
+
if sec['problems'] or sec['ref_missing']:
|
|
3685
|
+
extra_failed += bool(sec['problems']) + bool(sec['ref_missing'])
|
|
3686
|
+
print(' unrepairable descriptor damage / dangling references — '
|
|
3687
|
+
'chkdsk territory')
|
|
3688
|
+
elif sec_issues:
|
|
3689
|
+
print(' (repairable with --really: mirror fixes + $SII/$SDH rebuild; '
|
|
3690
|
+
'dangling references are report-only)')
|
|
3691
|
+
|
|
3692
|
+
if surface:
|
|
3693
|
+
print('stage 4: surface scan — reading every cluster ...')
|
|
3694
|
+
scan = run_surface(device, vol, direct)
|
|
3695
|
+
_print_surface(scan, indent=' ')
|
|
3696
|
+
if scan['bad']:
|
|
3697
|
+
extra_issues += 1
|
|
3698
|
+
if really:
|
|
3699
|
+
extra_failed += 1 # report-only: never repaired here
|
|
3700
|
+
|
|
3701
|
+
print('stage 5: verifying $Bitmap cluster accounting ...')
|
|
3702
|
+
# the shared survey's used-map is stale the moment any repair writes
|
|
3703
|
+
# (purge frees clusters, rebuild allocates INDX blocks) — recompute
|
|
3704
|
+
if really and (fixed + extra_fixed):
|
|
3705
|
+
audit = vol.cluster_audit()
|
|
3706
|
+
else:
|
|
3707
|
+
audit = vol.cluster_audit(survey=survey)
|
|
3708
|
+
for failure in audit['failures']:
|
|
3709
|
+
print(f'! {failure}')
|
|
3710
|
+
print(f' {audit["used_count"]} clusters referenced, '
|
|
3711
|
+
f'{audit["bitmap_count"]} marked in $Bitmap: '
|
|
3712
|
+
f'{audit["extra"]} leaked bit(s), {audit["missing"]} missing bit(s)')
|
|
3713
|
+
if audit['missing']:
|
|
3714
|
+
print(' WARNING: missing bits = live data on clusters marked free — '
|
|
3715
|
+
'writes to this volume are unsafe until repaired')
|
|
3716
|
+
if audit['extra'] or audit['missing']:
|
|
3717
|
+
extra_issues += 1
|
|
3718
|
+
if not really:
|
|
3719
|
+
print(' (repairable: --really rewrites $Bitmap with the computed map)')
|
|
3720
|
+
elif audit['failures']:
|
|
3721
|
+
extra_failed += 1
|
|
3722
|
+
print(' NOT rewriting $Bitmap — the computed map is incomplete '
|
|
3723
|
+
'(see failures above)')
|
|
3724
|
+
else:
|
|
3725
|
+
vol.apply_bitmap(audit)
|
|
3726
|
+
ondisk = vol._read_whole_attr(FILE_BITMAP, AT_DATA)
|
|
3727
|
+
n = len(audit['used_bytes'])
|
|
3728
|
+
mask = (1 << audit['nr_clusters']) - 1
|
|
3729
|
+
if (int.from_bytes(ondisk[:n], 'little') & mask
|
|
3730
|
+
== int.from_bytes(audit['used_bytes'], 'little') & mask):
|
|
3731
|
+
extra_fixed += 1
|
|
3732
|
+
print(' $Bitmap rewritten and verified')
|
|
3733
|
+
else:
|
|
3734
|
+
extra_failed += 1
|
|
3735
|
+
print(' VERIFY FAILED: $Bitmap readback does not match')
|
|
3736
|
+
|
|
3737
|
+
print('usn journal: verifying ...')
|
|
3738
|
+
usn = vol.usn_check()
|
|
3739
|
+
if not usn['present']:
|
|
3740
|
+
print(' none (journal disabled) — nothing to check')
|
|
3741
|
+
else:
|
|
3742
|
+
print(f' id {usn["journal_id"] or 0:#x}, valid range '
|
|
3743
|
+
f'[{usn["lowest"]}, {usn["next_usn"]}), {usn["records"]} records, '
|
|
3744
|
+
f'{len(usn["problems"])} problem(s)')
|
|
3745
|
+
for note in usn['notes']:
|
|
3746
|
+
print(f' note: {note}')
|
|
3747
|
+
for prob in usn['problems']:
|
|
3748
|
+
print(f'! {prob}')
|
|
3749
|
+
if usn['problems']:
|
|
3750
|
+
extra_issues += 1
|
|
3751
|
+
if not really:
|
|
3752
|
+
print(' (repairable: --really resets the journal; '
|
|
3753
|
+
'indexers/backup tools rescan)')
|
|
3754
|
+
else:
|
|
3755
|
+
try:
|
|
3756
|
+
vol.usn_reset()
|
|
3757
|
+
again = vol.usn_check()
|
|
3758
|
+
if again['problems'] or again['next_usn'] != 0:
|
|
3759
|
+
extra_failed += 1
|
|
3760
|
+
print(' RESET VERIFY FAILED')
|
|
3761
|
+
else:
|
|
3762
|
+
extra_fixed += 1
|
|
3763
|
+
print(' journal reset (new id; consumers will rescan)')
|
|
3764
|
+
except NtfsError as exc:
|
|
3765
|
+
extra_failed += 1
|
|
3766
|
+
print(f' RESET FAILED: {exc}')
|
|
3767
|
+
|
|
3768
|
+
print('view indexes: $Reparse / $ObjId ...')
|
|
3769
|
+
vw = vol.view_index_check(survey)
|
|
3770
|
+
for note in vw['notes']:
|
|
3771
|
+
print(f' note: {note}')
|
|
3772
|
+
print(f' {vw["checked"]} entr{"y" if vw["checked"] == 1 else "ies"} checked, '
|
|
3773
|
+
f'{len(vw["problems"])} problem(s)')
|
|
3774
|
+
for prob in vw['problems'][:10]:
|
|
3775
|
+
print(f'! {prob}')
|
|
3776
|
+
if vw['problems']:
|
|
3777
|
+
extra_issues += 1
|
|
3778
|
+
if really:
|
|
3779
|
+
extra_failed += 1
|
|
3780
|
+
print(' report-only (rebuild-from-records is a future repair)')
|
|
3781
|
+
|
|
3782
|
+
# -- crashed-session residue: the dirty flag + stale $LogFile --
|
|
3783
|
+
# chkdsk /f ends by resetting the log and clearing the dirty bit; a
|
|
3784
|
+
# volume left flagged is re-checked by Windows at boot and refused
|
|
3785
|
+
# outright by Linux ntfs3. Safe only once every finding above was
|
|
3786
|
+
# repaired — anything unresolved keeps the flag (and the recheck).
|
|
3787
|
+
# In --really mode the flag reads set because RawVolumeRW itself sets
|
|
3788
|
+
# it on open, so the pre-existing state is _was_dirty.
|
|
3789
|
+
print('dirty flag / $LogFile: ...')
|
|
3790
|
+
if (vol._was_dirty if really else vol._dirty_flag()):
|
|
3791
|
+
extra_issues += 1
|
|
3792
|
+
print('! volume is marked dirty (crashed session; $LogFile not clean)')
|
|
3793
|
+
unresolved = (dangling + torn + extra_issues - 1) - (fixed + extra_fixed)
|
|
3794
|
+
if not really:
|
|
3795
|
+
print(' (repairable: --really resets $LogFile and clears the '
|
|
3796
|
+
'flag once every finding above is repaired)')
|
|
3797
|
+
elif unresolved:
|
|
3798
|
+
extra_failed += 1
|
|
3799
|
+
print(f' left dirty: {unresolved} finding(s) unresolved — '
|
|
3800
|
+
'the next mount should still trigger a full check')
|
|
3801
|
+
else:
|
|
3802
|
+
vol.reset_logfile()
|
|
3803
|
+
vol.clear_dirty()
|
|
3804
|
+
if vol._dirty_flag():
|
|
3805
|
+
extra_failed += 1
|
|
3806
|
+
print(' CLEAR VERIFY FAILED')
|
|
3807
|
+
else:
|
|
3808
|
+
extra_fixed += 1
|
|
3809
|
+
print(' $LogFile reset (0xff) + dirty flag cleared — '
|
|
3810
|
+
'the volume mounts clean')
|
|
3811
|
+
else:
|
|
3812
|
+
print(' clean')
|
|
3813
|
+
|
|
3814
|
+
issues = dangling + torn + extra_issues
|
|
3815
|
+
fixed += extra_fixed
|
|
3816
|
+
print(f'-- {walked} directories walked: {dangling} dangling '
|
|
3817
|
+
f'entr{"y" if dangling == 1 else "ies"}, {torn} torn '
|
|
3818
|
+
f'ind{"ex" if torn == 1 else "exes"}'
|
|
3819
|
+
+ (f', {extra_issues} record/bitmap issue(s)' if extra_issues else '')
|
|
3820
|
+
+ (f'; {fixed} fixed, {issues - fixed} remaining' if really else ''))
|
|
3821
|
+
if health['torn']:
|
|
3822
|
+
print(f'note: {health["torn"]} MFT records are torn — their files are '
|
|
3823
|
+
'beyond index repair (chkdsk / restore from backup)')
|
|
3824
|
+
covered = ('records, runlists, indexes, security descriptors, $Bitmap and '
|
|
3825
|
+
'the USN journal are checked')
|
|
3826
|
+
print(f'note: {covered}'
|
|
3827
|
+
+ ('; the surface scan ran report-only' if surface
|
|
3828
|
+
else '; add --surface (or use /r) for the stage-4 read of every cluster'))
|
|
3829
|
+
if issues and not really:
|
|
3830
|
+
print('DRY RUN: re-run with --really to repair the findings listed above')
|
|
3831
|
+
return 1 if (issues if not really else issues - fixed) else 0
|
|
3832
|
+
|
|
3833
|
+
|
|
3834
|
+
# ── targeted pre-repair backup ───────────────────────────────────────────────
|
|
3835
|
+
|
|
3836
|
+
def do_backup(device: str, outdir: str, dirs: list[str]) -> int:
|
|
3837
|
+
'''Dump the boot sectors, the full raw $MFT, $MFTMirr, and each given
|
|
3838
|
+
directory's raw $I30 $INDEX_ALLOCATION — everything the repair commands
|
|
3839
|
+
can touch, restorable by hand. Metadata only; no file data.'''
|
|
3840
|
+
|
|
3841
|
+
os.makedirs(outdir, exist_ok=True)
|
|
3842
|
+
with open(device, 'rb') as dev, open(f'{outdir}/boot-16k.bin', 'wb') as fh:
|
|
3843
|
+
fh.write(dev.read(16384))
|
|
3844
|
+
print('boot : 16384 bytes')
|
|
3845
|
+
|
|
3846
|
+
with open_volume(device) as vol:
|
|
3847
|
+
rec_size = vol.mft_record_size()
|
|
3848
|
+
with open(f'{outdir}/mft.bin', 'wb') as fh:
|
|
3849
|
+
offset = 0
|
|
3850
|
+
while True:
|
|
3851
|
+
chunk = vol._mft_bulk(offset, 1 << 20)
|
|
3852
|
+
if not chunk:
|
|
3853
|
+
break
|
|
3854
|
+
fh.write(chunk)
|
|
3855
|
+
offset += len(chunk)
|
|
3856
|
+
print(f'$MFT : {offset} bytes ({offset // rec_size} records)')
|
|
3857
|
+
|
|
3858
|
+
mirror = vol._read_whole_attr(1, AT_DATA) # $MFTMirr
|
|
3859
|
+
with open(f'{outdir}/mftmirr.bin', 'wb') as fh:
|
|
3860
|
+
fh.write(mirror)
|
|
3861
|
+
print(f'$MFTMirr : {len(mirror)} bytes')
|
|
3862
|
+
|
|
3863
|
+
for path in dirs:
|
|
3864
|
+
mft_no = vol.resolve(path)
|
|
3865
|
+
total = -1
|
|
3866
|
+
try:
|
|
3867
|
+
blob = vol._read_whole_attr(mft_no, AT_INDEX_ALLOCATION, INDEX_I30, 4)
|
|
3868
|
+
with open(f'{outdir}/dir-{mft_no}-indx.bin', 'wb') as fh:
|
|
3869
|
+
fh.write(blob)
|
|
3870
|
+
total = len(blob)
|
|
3871
|
+
except NtfsError:
|
|
3872
|
+
pass # resident index only
|
|
3873
|
+
with open(f'{outdir}/dir-{mft_no}-path.txt', 'w') as fh:
|
|
3874
|
+
fh.write(path + '\n')
|
|
3875
|
+
state = '(resident index only)' if total < 0 else f'INDX {total} bytes'
|
|
3876
|
+
print(f'dir-{mft_no:<8}: {state} {path}')
|
|
3877
|
+
|
|
3878
|
+
total = 0
|
|
3879
|
+
with os.scandir(outdir) as it:
|
|
3880
|
+
entries = sorted((e for e in it if e.is_file() and e.name != 'SHA256SUMS'),
|
|
3881
|
+
key=lambda e: e.name)
|
|
3882
|
+
with open(f'{outdir}/SHA256SUMS', 'w') as sums:
|
|
3883
|
+
for entry in entries:
|
|
3884
|
+
digest = hashlib.sha256(open(entry.path, 'rb').read()).hexdigest()
|
|
3885
|
+
sums.write(f'{digest} {entry.name}\n')
|
|
3886
|
+
total += entry.stat().st_size # cached by scandir — no extra stat()
|
|
3887
|
+
print(f'backup complete: {total / 1e6:.0f} MB in {outdir}')
|
|
3888
|
+
return 0
|
|
3889
|
+
|
|
3890
|
+
|
|
3891
|
+
# ── partition discovery: `list` (raw table scan, nothing is mounted) ─────────
|
|
3892
|
+
|
|
3893
|
+
def _human(n: int) -> str:
|
|
3894
|
+
for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'):
|
|
3895
|
+
if n < 1024 or unit == 'TiB':
|
|
3896
|
+
return f'{n:.1f} {unit}' if unit != 'B' else f'{n} B'
|
|
3897
|
+
n /= 1024
|
|
3898
|
+
|
|
3899
|
+
|
|
3900
|
+
def _parse_partitions(fd) -> list[dict]:
|
|
3901
|
+
'''Parse the device's partitioning from raw sector reads: a bare NTFS
|
|
3902
|
+
volume (no table), an MBR (incl. the EBR chain of logical partitions),
|
|
3903
|
+
or a GPT behind its protective MBR. Returns [{index, offset, length,
|
|
3904
|
+
kind, name}]; nothing is mounted and nothing is written.'''
|
|
3905
|
+
sec0 = _pread(fd, 512, 0)
|
|
3906
|
+
if len(sec0) < 512:
|
|
3907
|
+
return []
|
|
3908
|
+
if sec0[3:7] == b'NTFS': # bare volume, no table
|
|
3909
|
+
bps = struct.unpack_from('<H', sec0, 11)[0] or 512
|
|
3910
|
+
sectors = struct.unpack_from('<Q', sec0, 40)[0]
|
|
3911
|
+
return [{'index': 0, 'offset': 0, 'length': (sectors + 1) * bps,
|
|
3912
|
+
'kind': 'bare', 'name': ''}]
|
|
3913
|
+
if sec0[510:512] != b'\x55\xaa':
|
|
3914
|
+
return []
|
|
3915
|
+
# entry: boot flag, CHS start (3), type, CHS end (3), start LBA, sectors
|
|
3916
|
+
entries = [(sec0[0x1BE + 16 * i + 4],
|
|
3917
|
+
*struct.unpack_from('<II', sec0, 0x1BE + 16 * i + 8))
|
|
3918
|
+
for i in range(4)] # (type, start_lba, sectors)
|
|
3919
|
+
if any(t == 0xEE for t, _s, _n in entries): # protective MBR → GPT
|
|
3920
|
+
hdr = _pread(fd, 512, 512)
|
|
3921
|
+
if hdr[:8] != b'EFI PART':
|
|
3922
|
+
return []
|
|
3923
|
+
arr_lba = struct.unpack_from('<Q', hdr, 72)[0]
|
|
3924
|
+
num, esize = struct.unpack_from('<II', hdr, 80)
|
|
3925
|
+
# clamp what a CORRUPT header can make us read: the spec minimum is
|
|
3926
|
+
# 128 entries of 128 bytes; nothing sane exceeds 512 entries
|
|
3927
|
+
if not 0 < esize <= 4096:
|
|
3928
|
+
return []
|
|
3929
|
+
num = min(num, 512)
|
|
3930
|
+
blob = _pread(fd, -(-num * esize // 512) * 512, arr_lba * 512)
|
|
3931
|
+
out = []
|
|
3932
|
+
for i in range(num):
|
|
3933
|
+
e = blob[i * esize:(i + 1) * esize]
|
|
3934
|
+
if len(e) < 128 or e[:16] == b'\x00' * 16:
|
|
3935
|
+
continue
|
|
3936
|
+
start, end = struct.unpack_from('<QQ', e, 32)
|
|
3937
|
+
name = e[56:128].decode('utf-16-le', 'replace').rstrip('\x00')
|
|
3938
|
+
out.append({'index': i + 1, 'offset': start * 512,
|
|
3939
|
+
'length': (end - start + 1) * 512,
|
|
3940
|
+
'kind': 'gpt', 'name': name})
|
|
3941
|
+
return out
|
|
3942
|
+
out = []
|
|
3943
|
+
for i, (ptype, start, sectors) in enumerate(entries):
|
|
3944
|
+
if not ptype:
|
|
3945
|
+
continue
|
|
3946
|
+
if ptype in (0x05, 0x0F, 0x85): # extended → walk the EBRs
|
|
3947
|
+
link, idx = 0, 5
|
|
3948
|
+
seen = set() # corrupt chains can cycle
|
|
3949
|
+
while link not in seen and len(seen) < 128:
|
|
3950
|
+
seen.add(link)
|
|
3951
|
+
ebr = _pread(fd, 512, (start + link) * 512)
|
|
3952
|
+
if len(ebr) < 512 or ebr[510:512] != b'\x55\xaa':
|
|
3953
|
+
break
|
|
3954
|
+
t0 = ebr[0x1BE + 4]
|
|
3955
|
+
s0, n0 = struct.unpack_from('<II', ebr, 0x1BE + 8)
|
|
3956
|
+
if t0:
|
|
3957
|
+
out.append({'index': idx,
|
|
3958
|
+
'offset': (start + link + s0) * 512,
|
|
3959
|
+
'length': n0 * 512, 'kind': 'mbr',
|
|
3960
|
+
'name': f'type {t0:#04x}'})
|
|
3961
|
+
idx += 1
|
|
3962
|
+
t1 = ebr[0x1BE + 16 + 4]
|
|
3963
|
+
s1 = struct.unpack_from('<I', ebr, 0x1BE + 16 + 8)[0]
|
|
3964
|
+
if not t1:
|
|
3965
|
+
break
|
|
3966
|
+
link = s1 # next EBR, extended-relative
|
|
3967
|
+
continue
|
|
3968
|
+
out.append({'index': i + 1, 'offset': start * 512,
|
|
3969
|
+
'length': sectors * 512, 'kind': 'mbr',
|
|
3970
|
+
'name': f'type {ptype:#04x}'})
|
|
3971
|
+
return out
|
|
3972
|
+
|
|
3973
|
+
|
|
3974
|
+
def _candidate_devices() -> list[str]:
|
|
3975
|
+
'''Whole-disk device nodes to scan when none were given. Read-only probes;
|
|
3976
|
+
missing/unopenable candidates are simply skipped.'''
|
|
3977
|
+
if sys.platform.startswith('linux'):
|
|
3978
|
+
try:
|
|
3979
|
+
return ['/dev/' + n for n in sorted(os.listdir('/sys/block'))
|
|
3980
|
+
if not n.startswith(('ram', 'zram'))]
|
|
3981
|
+
except OSError:
|
|
3982
|
+
return []
|
|
3983
|
+
if sys.platform == 'win32':
|
|
3984
|
+
return ['\\\\.\\PhysicalDrive' + str(i) for i in range(16)]
|
|
3985
|
+
if sys.platform == 'darwin':
|
|
3986
|
+
import re
|
|
3987
|
+
return ['/dev/' + n for n in sorted(os.listdir('/dev'))
|
|
3988
|
+
if re.fullmatch(r'disk\d+', n)]
|
|
3989
|
+
import re # the BSDs
|
|
3990
|
+
return ['/dev/' + n for n in sorted(os.listdir('/dev'))
|
|
3991
|
+
if re.fullmatch(r'(ada|da|vtbd|nvd|nda)\d+', n)]
|
|
3992
|
+
|
|
3993
|
+
|
|
3994
|
+
def list_volumes(devices: list[str] | None) -> int:
|
|
3995
|
+
'''`list`: find NTFS partitions by reading partition tables + boot sectors
|
|
3996
|
+
straight off the devices — nothing needs to be (or gets) mounted. With
|
|
3997
|
+
no arguments, scans the platform's whole-disk devices; arguments may be
|
|
3998
|
+
device nodes or image files.'''
|
|
3999
|
+
explicit = devices is not None
|
|
4000
|
+
found = 0
|
|
4001
|
+
for dev in (devices if explicit else _candidate_devices()):
|
|
4002
|
+
try:
|
|
4003
|
+
fd = os.open(dev, os.O_RDONLY | _O_BINARY)
|
|
4004
|
+
except FileNotFoundError:
|
|
4005
|
+
if explicit:
|
|
4006
|
+
print(f'{dev}: not found')
|
|
4007
|
+
continue
|
|
4008
|
+
except (PermissionError, OSError) as exc:
|
|
4009
|
+
print(f'{dev}: cannot open read-only ({exc.strerror or exc}) — '
|
|
4010
|
+
'root/admin is needed for raw device reads')
|
|
4011
|
+
continue
|
|
4012
|
+
try:
|
|
4013
|
+
parts = _parse_partitions(fd)
|
|
4014
|
+
finally:
|
|
4015
|
+
os.close(fd)
|
|
4016
|
+
if not parts:
|
|
4017
|
+
if explicit:
|
|
4018
|
+
print(f'{dev}: no NTFS boot sector and no partition table')
|
|
4019
|
+
continue
|
|
4020
|
+
for p in parts:
|
|
4021
|
+
try:
|
|
4022
|
+
boot = None
|
|
4023
|
+
fd = os.open(dev, os.O_RDONLY | _O_BINARY)
|
|
4024
|
+
try:
|
|
4025
|
+
boot = _pread(fd, 512, p['offset'])
|
|
4026
|
+
finally:
|
|
4027
|
+
os.close(fd)
|
|
4028
|
+
except OSError:
|
|
4029
|
+
continue
|
|
4030
|
+
if boot[3:7] != b'NTFS':
|
|
4031
|
+
continue
|
|
4032
|
+
found += 1
|
|
4033
|
+
state = 'unreadable'
|
|
4034
|
+
label = ''
|
|
4035
|
+
try:
|
|
4036
|
+
# the engine's own open-time dirty warning (stderr) is
|
|
4037
|
+
# redundant here — the row already says DIRTY
|
|
4038
|
+
import contextlib, io
|
|
4039
|
+
with contextlib.redirect_stderr(io.StringIO()):
|
|
4040
|
+
with RawVolume(dev, base=p['offset']) as v:
|
|
4041
|
+
label = v.volume_label()
|
|
4042
|
+
dirty = v._dirty_flag()
|
|
4043
|
+
state = ('DIRTY — a crashed session; /f repairs'
|
|
4044
|
+
if dirty else 'clean')
|
|
4045
|
+
except (NtfsError, OSError) as exc:
|
|
4046
|
+
state = f'DAMAGED ({exc}) — a repair candidate'
|
|
4047
|
+
where = ('' if p['kind'] == 'bare'
|
|
4048
|
+
else f' partition {p["index"]} @ {p["offset"]:#x}')
|
|
4049
|
+
name = f' [{p["name"]}]' if p['name'] and p['kind'] == 'gpt' else ''
|
|
4050
|
+
print(f'{dev}{where} ntfs '
|
|
4051
|
+
+ (f'{label!r} ' if label else '')
|
|
4052
|
+
+ f'{_human(p["length"])}{name} {state}')
|
|
4053
|
+
if not found:
|
|
4054
|
+
print('no NTFS volumes found' + ('' if explicit else
|
|
4055
|
+
' (no read access to any disk? try with root/admin)'))
|
|
4056
|
+
else:
|
|
4057
|
+
print(f'-- {found} NTFS volume(s); this listing mounted nothing and '
|
|
4058
|
+
'wrote nothing')
|
|
4059
|
+
return 0
|
|
4060
|
+
|
|
4061
|
+
|
|
4062
|
+
# ── CLI ──────────────────────────────────────────────────────────────────────
|
|
4063
|
+
|
|
4064
|
+
def main() -> int:
|
|
4065
|
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
4066
|
+
sub = parser.add_subparsers(dest='cmd', required=True)
|
|
4067
|
+
|
|
4068
|
+
p_scan = sub.add_parser('scan', help='read-only: list a directory and health-check every entry')
|
|
4069
|
+
p_scan.add_argument('device')
|
|
4070
|
+
p_scan.add_argument('dir')
|
|
4071
|
+
|
|
4072
|
+
p_rm = sub.add_parser('rm', help='remove ONE dangling index entry (dry-run unless --really)')
|
|
4073
|
+
p_rm.add_argument('device')
|
|
4074
|
+
p_rm.add_argument('dir')
|
|
4075
|
+
p_rm.add_argument('name')
|
|
4076
|
+
p_rm.add_argument('--really', action='store_true', help='actually write; default is dry-run')
|
|
4077
|
+
|
|
4078
|
+
p_inspect = sub.add_parser('inspect', help='read-only triage: free record, reused record, or true orphan?')
|
|
4079
|
+
p_inspect.add_argument('device')
|
|
4080
|
+
p_inspect.add_argument('dir')
|
|
4081
|
+
p_inspect.add_argument('name')
|
|
4082
|
+
|
|
4083
|
+
p_purge = sub.add_parser('purge', help='fully reclaim a TRUE ORPHAN: dirent + record + clusters (dry-run unless --really)')
|
|
4084
|
+
p_purge.add_argument('device')
|
|
4085
|
+
p_purge.add_argument('dir')
|
|
4086
|
+
p_purge.add_argument('name')
|
|
4087
|
+
p_purge.add_argument('--really', action='store_true', help='actually write; default is dry-run')
|
|
4088
|
+
|
|
4089
|
+
p_rebuild = sub.add_parser('rebuild', help='rebuild a TORN directory index from the MFT (dry-run unless --really)')
|
|
4090
|
+
p_rebuild.add_argument('device')
|
|
4091
|
+
p_rebuild.add_argument('dir')
|
|
4092
|
+
p_rebuild.add_argument('--really', action='store_true', help='actually write; default is dry-run')
|
|
4093
|
+
p_rebuild.add_argument('--force', action='store_true', help='rebuild even if the index currently reads fine')
|
|
4094
|
+
|
|
4095
|
+
p_fix = sub.add_parser('fix', aliases=['/f', '/F', '/r', '/R'],
|
|
4096
|
+
help='chkdsk /f flow: verify MFT records + runlists, walk '
|
|
4097
|
+
'EVERY directory index, validate $Secure, reconcile '
|
|
4098
|
+
'$Bitmap, check the USN journal (dry-run unless '
|
|
4099
|
+
'--really); /r or --surface adds the stage-4 read '
|
|
4100
|
+
'of every cluster')
|
|
4101
|
+
p_fix.add_argument('device')
|
|
4102
|
+
p_fix.add_argument('--really', action='store_true', help='actually write; default is dry-run')
|
|
4103
|
+
p_fix.add_argument('--surface', action='store_true',
|
|
4104
|
+
help='also read every cluster (chkdsk /r style, report-only)')
|
|
4105
|
+
p_fix.add_argument('--direct', action='store_true',
|
|
4106
|
+
help='surface scan with O_DIRECT (do not pollute the page cache)')
|
|
4107
|
+
|
|
4108
|
+
p_surface = sub.add_parser('surface', help='stage 4 only: read every cluster and '
|
|
4109
|
+
'attribute unreadable ones to their files '
|
|
4110
|
+
'(always report-only)')
|
|
4111
|
+
p_surface.add_argument('device')
|
|
4112
|
+
p_surface.add_argument('--direct', action='store_true',
|
|
4113
|
+
help='read with O_DIRECT (do not pollute the page cache)')
|
|
4114
|
+
|
|
4115
|
+
p_backup = sub.add_parser('backup', help='dump boot sectors, $MFT, $MFTMirr and given '
|
|
4116
|
+
"directories' raw $I30 indexes to a directory")
|
|
4117
|
+
p_backup.add_argument('device')
|
|
4118
|
+
p_backup.add_argument('outdir')
|
|
4119
|
+
p_backup.add_argument('dirs', nargs='*', help='directories whose $INDEX_ALLOCATION to dump')
|
|
4120
|
+
|
|
4121
|
+
p_usn = sub.add_parser('usn', help='validate the USN change journal; --really resets a '
|
|
4122
|
+
'corrupt journal (--reset forces the reset)')
|
|
4123
|
+
p_usn.add_argument('device')
|
|
4124
|
+
p_usn.add_argument('--really', action='store_true', help='actually write; default is dry-run')
|
|
4125
|
+
p_usn.add_argument('--reset', action='store_true', help='reset even if the journal is consistent')
|
|
4126
|
+
|
|
4127
|
+
p_secure = sub.add_parser('secure', help='validate $Secure: $SDS descriptors + mirrors, '
|
|
4128
|
+
'$SII/$SDH indexes, per-file references '
|
|
4129
|
+
'(--really repairs mirrors and rebuilds indexes)')
|
|
4130
|
+
p_secure.add_argument('device')
|
|
4131
|
+
p_secure.add_argument('--really', action='store_true', help='actually write; default is dry-run')
|
|
4132
|
+
|
|
4133
|
+
p_list = sub.add_parser('list', help='find NTFS partitions: raw partition-table + '
|
|
4134
|
+
'boot-sector scan of the local disks (or the '
|
|
4135
|
+
'given devices/images); mounts nothing')
|
|
4136
|
+
p_list.add_argument('devices', nargs='*',
|
|
4137
|
+
help='devices or image files to scan (default: all local disks)')
|
|
4138
|
+
|
|
4139
|
+
args = parser.parse_args()
|
|
4140
|
+
if args.cmd == 'list': # read-only by construction; may target mounted disks
|
|
4141
|
+
return list_volumes(args.devices or None)
|
|
4142
|
+
if args.cmd in ('/r', '/R'):
|
|
4143
|
+
args.surface = True
|
|
4144
|
+
if args.cmd in ('/f', '/F', '/r', '/R'):
|
|
4145
|
+
args.cmd = 'fix'
|
|
4146
|
+
assert_not_mounted(args.device)
|
|
4147
|
+
|
|
4148
|
+
if args.cmd == 'fix':
|
|
4149
|
+
return full_fix(args.device, args.really, args.surface, args.direct)
|
|
4150
|
+
|
|
4151
|
+
if args.cmd == 'surface':
|
|
4152
|
+
with open_volume(args.device, readonly=True) as volume:
|
|
4153
|
+
scan = run_surface(args.device, volume, args.direct)
|
|
4154
|
+
_print_surface(scan)
|
|
4155
|
+
return 1 if scan['bad'] else 0
|
|
4156
|
+
|
|
4157
|
+
if args.cmd == 'backup':
|
|
4158
|
+
return do_backup(args.device, args.outdir, args.dirs)
|
|
4159
|
+
|
|
4160
|
+
if args.cmd == 'secure':
|
|
4161
|
+
with open_volume(args.device, readonly=not args.really) as volume:
|
|
4162
|
+
chk = volume.secure_check()
|
|
4163
|
+
for note in chk['notes']:
|
|
4164
|
+
print(f'note: {note}')
|
|
4165
|
+
if not chk['present']:
|
|
4166
|
+
return 0
|
|
4167
|
+
print(f'descriptors: {chk["descriptors"]} in $SDS, '
|
|
4168
|
+
f'{chk["files_checked"]} files reference them')
|
|
4169
|
+
findings = (chk['problems'] + [f'mirror mismatch at {p} ({s} side is good)'
|
|
4170
|
+
for p, _l, s in chk['mirror_fixes']]
|
|
4171
|
+
+ chk['index_problems'] + chk['ref_missing'])
|
|
4172
|
+
for prob in findings:
|
|
4173
|
+
print(f'! {prob}')
|
|
4174
|
+
if not findings:
|
|
4175
|
+
print('$Secure consistent')
|
|
4176
|
+
return 0
|
|
4177
|
+
if not args.really:
|
|
4178
|
+
print('DRY RUN: re-run with --really to repair mirrors and rebuild '
|
|
4179
|
+
'$SII/$SDH (unrepairable damage and dangling references are '
|
|
4180
|
+
'report-only)')
|
|
4181
|
+
return 1
|
|
4182
|
+
if chk['mirror_fixes']:
|
|
4183
|
+
n = volume.secure_fix_mirrors(chk['mirror_fixes'])
|
|
4184
|
+
print(f'{n} $SDS mirror pair(s) repaired')
|
|
4185
|
+
if chk['index_problems']:
|
|
4186
|
+
try:
|
|
4187
|
+
result = volume.rebuild_secure_indexes(chk['sds_entries'])
|
|
4188
|
+
print(f'$SII/$SDH rebuilt from $SDS ({result["added"]} descriptors)')
|
|
4189
|
+
except NtfsError as exc:
|
|
4190
|
+
print(f'$SII/$SDH not rebuilt: {exc}')
|
|
4191
|
+
again = volume.secure_check()
|
|
4192
|
+
remaining = (again['problems'] + again['index_problems']
|
|
4193
|
+
+ [f'mirror mismatch at {p}' for p, _l, _s in again['mirror_fixes']]
|
|
4194
|
+
+ again['ref_missing'])
|
|
4195
|
+
print(f'verify: {len(remaining)} finding(s) remain')
|
|
4196
|
+
for prob in remaining[:10]:
|
|
4197
|
+
print(f'! {prob}')
|
|
4198
|
+
return 1 if remaining else 0
|
|
4199
|
+
|
|
4200
|
+
if args.cmd == 'usn':
|
|
4201
|
+
with open_volume(args.device, readonly=not args.really) as volume:
|
|
4202
|
+
info = volume.usn_check()
|
|
4203
|
+
if not info['present']:
|
|
4204
|
+
print('no USN journal on this volume (nothing to check)')
|
|
4205
|
+
return 0
|
|
4206
|
+
print(f'journal id : {info["journal_id"] or 0:#x}')
|
|
4207
|
+
print(f'valid range: [{info["lowest"]}, {info["next_usn"]}), '
|
|
4208
|
+
f'{info["records"]} records walked')
|
|
4209
|
+
for note in info['notes']:
|
|
4210
|
+
print(f'note: {note}')
|
|
4211
|
+
for prob in info['problems']:
|
|
4212
|
+
print(f'! {prob}')
|
|
4213
|
+
if not info['problems'] and not args.reset:
|
|
4214
|
+
print('journal consistent')
|
|
4215
|
+
return 0
|
|
4216
|
+
if not args.really:
|
|
4217
|
+
print('DRY RUN: re-run with --really to reset the journal '
|
|
4218
|
+
'(new id; indexers/backup tools rescan)')
|
|
4219
|
+
return 1 if info['problems'] else 0
|
|
4220
|
+
new_id = volume.usn_reset()
|
|
4221
|
+
check = volume.usn_check()
|
|
4222
|
+
ok = not check['problems'] and check['next_usn'] == 0
|
|
4223
|
+
print(f'journal reset: new id {new_id:#x}'
|
|
4224
|
+
+ ('' if ok else ' — VERIFY FAILED'))
|
|
4225
|
+
return 0 if ok else 1
|
|
4226
|
+
|
|
4227
|
+
if args.cmd == 'rebuild':
|
|
4228
|
+
with open_volume(args.device, readonly=True) as volume:
|
|
4229
|
+
probe = volume.probe_dir(args.dir)
|
|
4230
|
+
census = volume.children_from_mft(volume.resolve(args.dir))
|
|
4231
|
+
|
|
4232
|
+
if probe['readable']:
|
|
4233
|
+
print(f'index : readable, {len(probe["entries"])} entries '
|
|
4234
|
+
f'({sum(e["status"] != "ok" for e in probe["entries"])} dangling)')
|
|
4235
|
+
else:
|
|
4236
|
+
print(f'index : UNREADABLE (torn) — {probe["error"]}')
|
|
4237
|
+
print(f'mft census: {len(census["children"])} names claim this directory as parent '
|
|
4238
|
+
f'({census["stale_parent"]} stale-parent names and '
|
|
4239
|
+
f'{census["unreadable_records"]} torn MFT records skipped)')
|
|
4240
|
+
for child in census['children']:
|
|
4241
|
+
kind = {1: 'win32', 2: 'dos', 3: 'win32+dos'}.get(child['type'], 'posix')
|
|
4242
|
+
print(f' {child["name"]!r:60} mft={child["record"]} seq={child["seq"]} [{kind}]')
|
|
4243
|
+
|
|
4244
|
+
if probe['readable'] and not args.force:
|
|
4245
|
+
print('index reads fine — refusing to rebuild without --force')
|
|
4246
|
+
return 1
|
|
4247
|
+
if not args.really:
|
|
4248
|
+
print('DRY RUN: re-run with --really to wipe the $I30 index and rebuild it from these names')
|
|
4249
|
+
return 0
|
|
4250
|
+
|
|
4251
|
+
with open_volume(args.device, readonly=False) as volume:
|
|
4252
|
+
result = volume.rebuild_dir(args.dir)
|
|
4253
|
+
print(f'rebuilt: {result["added"]} entries re-added '
|
|
4254
|
+
f'({result["duplicates"]} duplicates skipped, {result["planned"]} planned)')
|
|
4255
|
+
|
|
4256
|
+
with open_volume(args.device, readonly=True) as volume:
|
|
4257
|
+
verify = volume.probe_dir(args.dir)
|
|
4258
|
+
if not verify['readable']:
|
|
4259
|
+
print(f'VERIFY FAILED: index still unreadable — {verify["error"]}')
|
|
4260
|
+
return 1
|
|
4261
|
+
bad = [e for e in verify['entries'] if e['status'] != 'ok']
|
|
4262
|
+
print(f'verify : index readable, {len(verify["entries"])} entries, {len(bad)} dangling')
|
|
4263
|
+
for entry in bad:
|
|
4264
|
+
print(f' still dangling: {entry["name"]!r} — triage with inspect')
|
|
4265
|
+
return 0
|
|
4266
|
+
|
|
4267
|
+
if args.cmd == 'inspect':
|
|
4268
|
+
with open_volume(args.device, readonly=True) as volume:
|
|
4269
|
+
verdict = volume.classify_dirent(args.dir, args.name)
|
|
4270
|
+
print(f'entry : {verdict["name"]!r} in {args.dir!r} '
|
|
4271
|
+
f'-> mft={verdict["dirent_record"]} seq={verdict["dirent_seq"]}')
|
|
4272
|
+
record = verdict.get('record_info')
|
|
4273
|
+
if record and record.get('open'):
|
|
4274
|
+
print(f'record : seq={record["seq"]} in_use={record["in_use"]} '
|
|
4275
|
+
f'links={record["link_count"]} dir={record["is_dir"]}')
|
|
4276
|
+
for entry in record['names']:
|
|
4277
|
+
print(f' claims : {entry["name"]!r} (parent mft={entry["parent_record"]})')
|
|
4278
|
+
elif record:
|
|
4279
|
+
print(f'record : unreadable/free ({record["error"]})')
|
|
4280
|
+
print(f'state : {verdict["state"]}')
|
|
4281
|
+
print(f'action : {verdict["action"]}')
|
|
4282
|
+
return 0 if verdict['state'] == 'healthy' else 1
|
|
4283
|
+
|
|
4284
|
+
if args.cmd == 'purge':
|
|
4285
|
+
with open_volume(args.device, readonly=not args.really) as volume:
|
|
4286
|
+
verdict = volume.purge_orphan(args.dir, args.name, really=args.really)
|
|
4287
|
+
if args.really:
|
|
4288
|
+
print(f'purged {args.name!r}: dirent removed, record '
|
|
4289
|
+
f'{verdict["dirent_record"]} and its clusters freed')
|
|
4290
|
+
else:
|
|
4291
|
+
print(f'DRY RUN: {args.name!r} is a true orphan (record {verdict["dirent_record"]}); '
|
|
4292
|
+
're-run with --really to reclaim it fully')
|
|
4293
|
+
return 0
|
|
4294
|
+
|
|
4295
|
+
if args.cmd == 'scan':
|
|
4296
|
+
with open_volume(args.device, readonly=True) as volume:
|
|
4297
|
+
try:
|
|
4298
|
+
entries = volume.scan_dir(args.dir)
|
|
4299
|
+
except NtfsError as exc:
|
|
4300
|
+
print(f'index walk failed: {exc}')
|
|
4301
|
+
print('the directory index itself is unreadable (torn INDX block?) — '
|
|
4302
|
+
'triage with `rebuild` (dry-run first)')
|
|
4303
|
+
return 2
|
|
4304
|
+
broken = 0
|
|
4305
|
+
for entry in entries:
|
|
4306
|
+
if entry['status'] == 'dir-self':
|
|
4307
|
+
continue
|
|
4308
|
+
flag = ' ' if entry['status'] == 'ok' else '!'
|
|
4309
|
+
print(f'{flag} {entry["name"]!r:60} mft={entry["record"]} seq={entry["seq"]} {entry["status"]}')
|
|
4310
|
+
broken += entry['status'] != 'ok'
|
|
4311
|
+
print(f'-- {broken} dangling entr{"y" if broken == 1 else "ies"}')
|
|
4312
|
+
return 1 if broken else 0
|
|
4313
|
+
|
|
4314
|
+
if not args.really:
|
|
4315
|
+
with open_volume(args.device, readonly=True) as volume:
|
|
4316
|
+
found = volume.remove_dirent(args.dir, args.name, really=False)
|
|
4317
|
+
print(f'DRY RUN: would remove {found["name"]!r} -> mft={found["record"]} seq={found["seq"]}')
|
|
4318
|
+
print('re-run with --really to write')
|
|
4319
|
+
return 0
|
|
4320
|
+
|
|
4321
|
+
with open_volume(args.device, readonly=False) as volume:
|
|
4322
|
+
found = volume.remove_dirent(args.dir, args.name, really=True)
|
|
4323
|
+
print(f'removed {found["name"]!r} (was mft={found["record"]} seq={found["seq"]})')
|
|
4324
|
+
return 0
|
|
4325
|
+
|
|
4326
|
+
|
|
4327
|
+
if __name__ == '__main__':
|
|
4328
|
+
sys.exit(main())
|