diskstack 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diskstack/__init__.py +5 -0
- diskstack/_vendor/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/COPYING +24 -0
- diskstack/_vendor/greaseweazle/__init__.py +2 -0
- diskstack/_vendor/greaseweazle/codec/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/amiga/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/amiga/amigados.py +221 -0
- diskstack/_vendor/greaseweazle/codec/apple2/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/apple2/apple2_gcr.py +223 -0
- diskstack/_vendor/greaseweazle/codec/bitcell.py +117 -0
- diskstack/_vendor/greaseweazle/codec/codec.py +362 -0
- diskstack/_vendor/greaseweazle/codec/commodore/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/commodore/c64_gcr.py +229 -0
- diskstack/_vendor/greaseweazle/codec/datageneral/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/datageneral/datageneral.py +211 -0
- diskstack/_vendor/greaseweazle/codec/hp/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/hp/hp_mmfm.py +258 -0
- diskstack/_vendor/greaseweazle/codec/ibm/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/ibm/ibm.py +1005 -0
- diskstack/_vendor/greaseweazle/codec/macintosh/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/macintosh/mac_gcr.py +238 -0
- diskstack/_vendor/greaseweazle/codec/micropolis/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/micropolis/micropolis.py +233 -0
- diskstack/_vendor/greaseweazle/codec/northstar/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/codec/northstar/northstar.py +210 -0
- diskstack/_vendor/greaseweazle/data/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/data/diskdefs.cfg +81 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_acorn.cfg +119 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_akai.cfg +27 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_amiga.cfg +17 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_apple2.cfg +31 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_apricot.cfg +25 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_atari.cfg +33 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_atarist.cfg +76 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_coco.cfg +98 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_commodore.cfg +127 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_datageneral.cfg +8 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_dec.cfg +37 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_dragon.cfg +61 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_eagle.cfg +25 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_ensoniq.cfg +39 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_epson.cfg +95 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_gem.cfg +15 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_hp.cfg +25 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_ibm.cfg +140 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_kaypro.cfg +58 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_luxor.cfg +96 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_mac.cfg +71 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_micropolis.cfg +73 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_mm1.cfg +69 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_msx.cfg +45 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_northstar.cfg +37 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_occ1.cfg +23 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_olivetti.cfg +18 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_pc98.cfg +69 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_raw.cfg +25 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_rm.cfg +98 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_sci.cfg +15 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_sega.cfg +13 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_sharp.cfg +13 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_thomson.cfg +78 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_tsc.cfg +81 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_xerox.cfg +59 -0
- diskstack/_vendor/greaseweazle/data/diskdefs_zx.cfg +245 -0
- diskstack/_vendor/greaseweazle/error.py +19 -0
- diskstack/_vendor/greaseweazle/flux.py +315 -0
- diskstack/_vendor/greaseweazle/image/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/image/a2r.py +133 -0
- diskstack/_vendor/greaseweazle/image/acorn.py +28 -0
- diskstack/_vendor/greaseweazle/image/adf.py +15 -0
- diskstack/_vendor/greaseweazle/image/apple2.py +18 -0
- diskstack/_vendor/greaseweazle/image/apridisk.py +107 -0
- diskstack/_vendor/greaseweazle/image/caps.py +464 -0
- diskstack/_vendor/greaseweazle/image/d64.py +51 -0
- diskstack/_vendor/greaseweazle/image/d81.py +28 -0
- diskstack/_vendor/greaseweazle/image/d88.py +211 -0
- diskstack/_vendor/greaseweazle/image/dcp.py +45 -0
- diskstack/_vendor/greaseweazle/image/dim.py +53 -0
- diskstack/_vendor/greaseweazle/image/dmk.py +166 -0
- diskstack/_vendor/greaseweazle/image/dsk.py +31 -0
- diskstack/_vendor/greaseweazle/image/edsk.py +591 -0
- diskstack/_vendor/greaseweazle/image/fd.py +23 -0
- diskstack/_vendor/greaseweazle/image/fdi.py +41 -0
- diskstack/_vendor/greaseweazle/image/hdm.py +15 -0
- diskstack/_vendor/greaseweazle/image/hfe.py +742 -0
- diskstack/_vendor/greaseweazle/image/image.py +130 -0
- diskstack/_vendor/greaseweazle/image/imd.py +201 -0
- diskstack/_vendor/greaseweazle/image/img.py +111 -0
- diskstack/_vendor/greaseweazle/image/kryoflux.py +304 -0
- diskstack/_vendor/greaseweazle/image/mgt.py +15 -0
- diskstack/_vendor/greaseweazle/image/msa.py +153 -0
- diskstack/_vendor/greaseweazle/image/nfd.py +113 -0
- diskstack/_vendor/greaseweazle/image/nsi.py +47 -0
- diskstack/_vendor/greaseweazle/image/scp.py +471 -0
- diskstack/_vendor/greaseweazle/image/sf7.py +15 -0
- diskstack/_vendor/greaseweazle/image/sharp2d.py +15 -0
- diskstack/_vendor/greaseweazle/image/td0.py +141 -0
- diskstack/_vendor/greaseweazle/image/xdf.py +15 -0
- diskstack/_vendor/greaseweazle/optimised/__init__.py +22 -0
- diskstack/_vendor/greaseweazle/tools/__init__.py +0 -0
- diskstack/_vendor/greaseweazle/tools/util.py +133 -0
- diskstack/_vendor/greaseweazle/track.py +494 -0
- diskstack/cache.py +210 -0
- diskstack/candidates.py +396 -0
- diskstack/cli.py +348 -0
- diskstack/errors.py +5 -0
- diskstack/filler.py +29 -0
- diskstack/formats.py +381 -0
- diskstack/parallel.py +73 -0
- diskstack/report.py +353 -0
- diskstack/stack.py +317 -0
- diskstack-1.0.0.dist-info/METADATA +329 -0
- diskstack-1.0.0.dist-info/RECORD +119 -0
- diskstack-1.0.0.dist-info/WHEEL +5 -0
- diskstack-1.0.0.dist-info/entry_points.txt +2 -0
- diskstack-1.0.0.dist-info/licenses/LICENSE +21 -0
- diskstack-1.0.0.dist-info/licenses/NOTICE +55 -0
- diskstack-1.0.0.dist-info/licenses/diskstack/_vendor/greaseweazle/COPYING +24 -0
- diskstack-1.0.0.dist-info/top_level.txt +1 -0
diskstack/report.py
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
"""The JSON report, the stdout tables, and the re-read command.
|
|
2
|
+
|
|
3
|
+
The re-read line is the point of the tool: it names only the tracks that are
|
|
4
|
+
still bad, so the next pass over the disk is seconds of drive time instead of
|
|
5
|
+
minutes, and the result goes straight back in as another input.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import datetime
|
|
11
|
+
import json
|
|
12
|
+
from collections import defaultdict
|
|
13
|
+
from dataclasses import asdict, dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Dict, List, Optional, Sequence
|
|
16
|
+
|
|
17
|
+
from diskstack import __version__
|
|
18
|
+
from diskstack.candidates import SourceInfo
|
|
19
|
+
from diskstack.stack import CLEAN, MISSING, UNRESOLVED, VOTED, StackResult
|
|
20
|
+
|
|
21
|
+
SCHEMA_VERSION = 1
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Progress:
|
|
26
|
+
"""How a merge moved against the report from the run before it."""
|
|
27
|
+
|
|
28
|
+
recovered: int
|
|
29
|
+
lost: int
|
|
30
|
+
still_bad: int
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def read_previous(path: Path) -> Optional[dict]:
|
|
34
|
+
"""The report already sitting at ``path``, if it is one diskstack wrote."""
|
|
35
|
+
try:
|
|
36
|
+
previous = json.loads(path.read_text(encoding='utf-8'))
|
|
37
|
+
except (OSError, ValueError):
|
|
38
|
+
return None
|
|
39
|
+
return previous if isinstance(previous, dict) else None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def compare(previous: dict, result: StackResult) -> Optional[Progress]:
|
|
43
|
+
"""What this merge changed since ``previous``, or None if not comparable.
|
|
44
|
+
|
|
45
|
+
The two runs have to cover the same sectors. Otherwise the report is of
|
|
46
|
+
another disk and a count of what changed would mean nothing.
|
|
47
|
+
"""
|
|
48
|
+
if previous.get('schema') != SCHEMA_VERSION:
|
|
49
|
+
return None
|
|
50
|
+
try:
|
|
51
|
+
was = {(s['cyl'], s['head'], s['sec_id']):
|
|
52
|
+
s['status'] in (CLEAN, VOTED)
|
|
53
|
+
for s in previous['sectors']}
|
|
54
|
+
except (KeyError, TypeError):
|
|
55
|
+
return None
|
|
56
|
+
now = {res.key: res.resolved for res in result.resolutions}
|
|
57
|
+
if set(was) != set(now):
|
|
58
|
+
return None
|
|
59
|
+
return Progress(
|
|
60
|
+
recovered=sum(1 for k, ok in now.items() if ok and not was[k]),
|
|
61
|
+
lost=sum(1 for k, ok in now.items() if not ok and was[k]),
|
|
62
|
+
still_bad=sum(1 for k, ok in now.items() if not ok and not was[k]))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def progress_note(progress: Optional[Progress]) -> List[str]:
|
|
66
|
+
"""The one line that says whether the last re-read was worth doing."""
|
|
67
|
+
if progress is None:
|
|
68
|
+
return []
|
|
69
|
+
if progress.recovered or progress.lost:
|
|
70
|
+
parts = [f'{progress.recovered} recovered']
|
|
71
|
+
if progress.lost:
|
|
72
|
+
parts.append(f'{progress.lost} lost')
|
|
73
|
+
return [f'Since the last report: {", ".join(parts)}.']
|
|
74
|
+
if progress.still_bad:
|
|
75
|
+
return [f'No change since the last report: the same '
|
|
76
|
+
f'{progress.still_bad} sectors are still bad. Another pass at '
|
|
77
|
+
f'the same settings may not do better; --pll is the other '
|
|
78
|
+
f'thing to vary.']
|
|
79
|
+
return []
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _ranges(values: Sequence[int]) -> str:
|
|
83
|
+
"""Collapse sorted integers into Greaseweazle range syntax: 3,7-9."""
|
|
84
|
+
parts, start, prev = [], None, None
|
|
85
|
+
for value in values:
|
|
86
|
+
if start is None:
|
|
87
|
+
start = prev = value
|
|
88
|
+
elif value == prev + 1:
|
|
89
|
+
prev = value
|
|
90
|
+
else:
|
|
91
|
+
parts.append(str(start) if start == prev else f'{start}-{prev}')
|
|
92
|
+
start = prev = value
|
|
93
|
+
if start is not None:
|
|
94
|
+
parts.append(str(start) if start == prev else f'{start}-{prev}')
|
|
95
|
+
return ','.join(parts)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def reread_trackspecs(result: StackResult) -> List[str]:
|
|
99
|
+
"""Greaseweazle ``--tracks=`` specs covering every unresolved sector.
|
|
100
|
+
|
|
101
|
+
Heads whose bad cylinders are identical share one spec; otherwise each
|
|
102
|
+
head gets its own, so the user never re-reads a track that is already good.
|
|
103
|
+
"""
|
|
104
|
+
per_head: Dict[int, set] = defaultdict(set)
|
|
105
|
+
for res in result.resolutions:
|
|
106
|
+
if not res.resolved:
|
|
107
|
+
per_head[res.head].add(res.cyl)
|
|
108
|
+
grouped: Dict[frozenset, List[int]] = defaultdict(list)
|
|
109
|
+
for head, cyls in per_head.items():
|
|
110
|
+
grouped[frozenset(cyls)].append(head)
|
|
111
|
+
specs = []
|
|
112
|
+
for cyls, heads in sorted(grouped.items(),
|
|
113
|
+
key=lambda kv: (sorted(kv[1]), sorted(kv[0]))):
|
|
114
|
+
specs.append(f'c={_ranges(sorted(cyls))}:h={_ranges(sorted(heads))}')
|
|
115
|
+
return specs
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def reread_commands(result: StackResult, name: str = 'retry.scp') -> List[str]:
|
|
119
|
+
"""Full ``gw read`` command lines for the tracks still missing data."""
|
|
120
|
+
specs = reread_trackspecs(result)
|
|
121
|
+
if len(specs) == 1:
|
|
122
|
+
return [f'gw read {name} --tracks={specs[0]}']
|
|
123
|
+
stem, suffix = name.rsplit('.', 1) if '.' in name else (name, 'scp')
|
|
124
|
+
return [f'gw read {stem}{i}.{suffix} --tracks={spec}'
|
|
125
|
+
for i, spec in enumerate(specs, 1)]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def summary_line(result: StackResult) -> str:
|
|
129
|
+
"""The headline: how much of the disk came out readable, and from where."""
|
|
130
|
+
counts = result.counts()
|
|
131
|
+
total = len(result.resolutions)
|
|
132
|
+
files = len({path for path, n in result.supplied.items() if n})
|
|
133
|
+
parts = [f'{counts[CLEAN]} clean '
|
|
134
|
+
f'(from {files} file{"s" if files != 1 else ""})']
|
|
135
|
+
if counts[VOTED]:
|
|
136
|
+
parts.append(f'{counts[VOTED]} recovered by vote')
|
|
137
|
+
if counts[UNRESOLVED]:
|
|
138
|
+
parts.append(f'{counts[UNRESOLVED]} unresolved')
|
|
139
|
+
if counts[MISSING]:
|
|
140
|
+
parts.append(f'{counts[MISSING]} missing')
|
|
141
|
+
return f'{total} sectors: ' + ', '.join(parts)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def render_table(headers: Sequence[str],
|
|
145
|
+
rows: Sequence[Sequence[object]],
|
|
146
|
+
align: str = '') -> str:
|
|
147
|
+
"""Fixed-width table; ``align`` is one of 'l'/'r' per column."""
|
|
148
|
+
cells = [[str(c) for c in row] for row in rows]
|
|
149
|
+
widths = [max(len(h), *(len(r[i]) for r in cells)) if cells else len(h)
|
|
150
|
+
for i, h in enumerate(headers)]
|
|
151
|
+
align = (align + 'l' * len(headers))[:len(headers)]
|
|
152
|
+
|
|
153
|
+
def line(values):
|
|
154
|
+
out = []
|
|
155
|
+
for value, width, how in zip(values, widths, align, strict=True):
|
|
156
|
+
out.append(value.rjust(width) if how == 'r' else value.ljust(width))
|
|
157
|
+
return ' ' + ' '.join(out).rstrip()
|
|
158
|
+
|
|
159
|
+
return '\n'.join([line(headers)] + [line(r) for r in cells])
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def source_table(sources: Sequence[SourceInfo], result: StackResult) -> str:
|
|
163
|
+
"""Per-input-file provenance: what each dump brought to the merge."""
|
|
164
|
+
rows = []
|
|
165
|
+
for info in sources:
|
|
166
|
+
rows.append([
|
|
167
|
+
info.path.name, info.kind, info.revolutions, info.candidates,
|
|
168
|
+
info.good, result.supplied.get(info.path, 0),
|
|
169
|
+
result.sole_source.get(info.path, 0),
|
|
170
|
+
])
|
|
171
|
+
return render_table(
|
|
172
|
+
['file', 'type', 'revs', 'attempts', 'good', 'supplied', 'sole'],
|
|
173
|
+
rows, align='llrrrrr')
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _track_rows(result: StackResult) -> List[List[object]]:
|
|
177
|
+
rows = []
|
|
178
|
+
for (cyl, head), sectors in sorted(result.by_track().items()):
|
|
179
|
+
counts = {CLEAN: 0, VOTED: 0, UNRESOLVED: 0, MISSING: 0}
|
|
180
|
+
for res in sectors:
|
|
181
|
+
counts[res.status] += 1
|
|
182
|
+
if not counts[UNRESOLVED] and not counts[MISSING]:
|
|
183
|
+
continue
|
|
184
|
+
bad = sorted(r.sec_id for r in sectors if not r.resolved)
|
|
185
|
+
rows.append([cyl, head, counts[CLEAN], counts[VOTED],
|
|
186
|
+
counts[UNRESOLVED], counts[MISSING],
|
|
187
|
+
','.join(str(s) for s in bad)])
|
|
188
|
+
return rows
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def track_table(result: StackResult) -> str:
|
|
192
|
+
"""Tracks that still hold a sector diskstack could not confirm."""
|
|
193
|
+
return render_table(
|
|
194
|
+
['cyl', 'head', 'clean', 'voted', 'unresolved', 'missing',
|
|
195
|
+
'sector ids still bad'],
|
|
196
|
+
_track_rows(result), align='rrrrrrl')
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def build(result: StackResult, sources: Sequence[SourceInfo],
|
|
200
|
+
fmt_name: str, fmt_detail: str, output: Path,
|
|
201
|
+
retry_name: str = 'retry.scp',
|
|
202
|
+
progress: Optional[Progress] = None) -> dict:
|
|
203
|
+
"""The machine-readable report: every sector, with its provenance."""
|
|
204
|
+
counts = result.counts()
|
|
205
|
+
return {
|
|
206
|
+
'schema': SCHEMA_VERSION,
|
|
207
|
+
'tool': 'diskstack',
|
|
208
|
+
'version': __version__,
|
|
209
|
+
'generated': datetime.datetime.now(datetime.timezone.utc)
|
|
210
|
+
.replace(microsecond=0).isoformat(),
|
|
211
|
+
'format': fmt_name,
|
|
212
|
+
'format_detail': fmt_detail,
|
|
213
|
+
'output': str(output),
|
|
214
|
+
'inputs': [
|
|
215
|
+
{
|
|
216
|
+
'path': str(info.path),
|
|
217
|
+
'type': info.kind,
|
|
218
|
+
'revolutions': info.revolutions,
|
|
219
|
+
'attempts': info.candidates,
|
|
220
|
+
'good': info.good,
|
|
221
|
+
'supplied': result.supplied.get(info.path, 0),
|
|
222
|
+
'sole_source': result.sole_source.get(info.path, 0),
|
|
223
|
+
'unexpected_sectors': len(info.unexpected),
|
|
224
|
+
'bytes': info.size,
|
|
225
|
+
'expected_bytes': info.expected_size or None,
|
|
226
|
+
'cached': info.cached,
|
|
227
|
+
}
|
|
228
|
+
for info in sources
|
|
229
|
+
],
|
|
230
|
+
'totals': {
|
|
231
|
+
'sectors': len(result.resolutions),
|
|
232
|
+
'clean': counts[CLEAN],
|
|
233
|
+
'recovered_by_vote': counts[VOTED],
|
|
234
|
+
'unresolved': counts[UNRESOLVED],
|
|
235
|
+
'missing': counts[MISSING],
|
|
236
|
+
'verified': sum(1 for r in result.resolutions if r.verified),
|
|
237
|
+
'contested': sum(1 for r in result.resolutions if r.contested),
|
|
238
|
+
},
|
|
239
|
+
'sectors': [
|
|
240
|
+
{
|
|
241
|
+
'cyl': res.cyl,
|
|
242
|
+
'head': res.head,
|
|
243
|
+
'sec_id': res.sec_id,
|
|
244
|
+
'size': res.size,
|
|
245
|
+
'status': res.status,
|
|
246
|
+
'attempts': res.attempts,
|
|
247
|
+
'good': res.good,
|
|
248
|
+
'agreement': res.agreement,
|
|
249
|
+
'discarded': res.discarded,
|
|
250
|
+
'unstable': res.unstable,
|
|
251
|
+
'contested': res.contested,
|
|
252
|
+
'verified': res.verified,
|
|
253
|
+
'method': res.method or None,
|
|
254
|
+
'sources': [{'path': str(c.source), 'rev': c.rev}
|
|
255
|
+
for c in res.sources],
|
|
256
|
+
}
|
|
257
|
+
for res in result.resolutions
|
|
258
|
+
],
|
|
259
|
+
'reread': {
|
|
260
|
+
'tracks': reread_trackspecs(result),
|
|
261
|
+
'commands': reread_commands(result, retry_name),
|
|
262
|
+
},
|
|
263
|
+
'since_last_report': asdict(progress) if progress else None,
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def write(report: dict, path: Path) -> None:
|
|
268
|
+
path.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def warnings(sources: Sequence[SourceInfo], fmt_name: str) -> List[str]:
|
|
272
|
+
"""What the user should know about the inputs before trusting the merge."""
|
|
273
|
+
out = []
|
|
274
|
+
for info in sources:
|
|
275
|
+
if n := len(info.unexpected):
|
|
276
|
+
out.append(f'{info.path.name}: ignored {n} sector'
|
|
277
|
+
f'{"s" if n != 1 else ""} the format does not expect')
|
|
278
|
+
if info.wrong_size:
|
|
279
|
+
tail = ('the extra sectors were ignored'
|
|
280
|
+
if info.size > info.expected_size
|
|
281
|
+
else 'the missing tail counts as unread')
|
|
282
|
+
out.append(f'{info.path.name}: {info.size} bytes where {fmt_name} '
|
|
283
|
+
f'is {info.expected_size}, so {tail}. Pass --format if '
|
|
284
|
+
f'that is the wrong disk format.')
|
|
285
|
+
return out
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def cache_note(sources: Sequence[SourceInfo]) -> List[str]:
|
|
289
|
+
"""Say when a run reused a stored decode rather than reading the flux."""
|
|
290
|
+
reused = sum(1 for info in sources if info.cached)
|
|
291
|
+
if not reused:
|
|
292
|
+
return []
|
|
293
|
+
return [f'{reused} of {len(sources)} inputs came out of the cache instead '
|
|
294
|
+
f'of being decoded again. --no-cache turns that off.']
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def contested_note(result: StackResult) -> List[str]:
|
|
298
|
+
"""Sectors where the dumps disagreed and every reading looked good.
|
|
299
|
+
|
|
300
|
+
Nothing arbitrates this: the merge keeps the best-ranked dump's copy. It is
|
|
301
|
+
what deliberate weak bits look like, and what two dumps of two different
|
|
302
|
+
disks look like.
|
|
303
|
+
"""
|
|
304
|
+
bad = [r for r in result.resolutions if r.contested]
|
|
305
|
+
if not bad:
|
|
306
|
+
return []
|
|
307
|
+
where = ', '.join(f'c{r.cyl}:h{r.head}:s{r.sec_id}' for r in bad[:4])
|
|
308
|
+
if len(bad) > 4:
|
|
309
|
+
where += f', and {len(bad) - 4} more'
|
|
310
|
+
return [f'{len(bad)} sector{"s" if len(bad) != 1 else ""} came out good '
|
|
311
|
+
f'in more than one dump but with different bytes ({where}). '
|
|
312
|
+
f"diskstack kept the best-ranked dump's copy and flagged them "
|
|
313
|
+
f'contested in the report.']
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def render(result: StackResult, sources: Sequence[SourceInfo],
|
|
317
|
+
fmt_name: str, fmt_detail: str,
|
|
318
|
+
retry_name: str = 'retry.scp',
|
|
319
|
+
tables: bool = True,
|
|
320
|
+
progress: Optional[Progress] = None) -> str:
|
|
321
|
+
"""The stdout view: summary, provenance, and what to re-read."""
|
|
322
|
+
out = ([f'Format: {fmt_name} ({fmt_detail})', '', summary_line(result)]
|
|
323
|
+
+ progress_note(progress))
|
|
324
|
+
if tables:
|
|
325
|
+
out += ['', 'Sources', source_table(sources, result)]
|
|
326
|
+
rows = _track_rows(result)
|
|
327
|
+
if rows:
|
|
328
|
+
out += ['', 'Tracks needing attention', track_table(result)]
|
|
329
|
+
notes = (warnings(sources, fmt_name) + cache_note(sources)
|
|
330
|
+
+ contested_note(result))
|
|
331
|
+
if notes:
|
|
332
|
+
out += [''] + notes
|
|
333
|
+
commands = reread_commands(result, retry_name)
|
|
334
|
+
if commands:
|
|
335
|
+
out += ['', 'Re-read just these tracks, then run diskstack again '
|
|
336
|
+
'with the new capture added:']
|
|
337
|
+
out += [' ' + c for c in commands]
|
|
338
|
+
shaky = sum(1 for r in result.resolutions
|
|
339
|
+
if not r.resolved and r.unstable)
|
|
340
|
+
if shaky:
|
|
341
|
+
out.append(f'{shaky} of those sectors read differently on every '
|
|
342
|
+
f'pass of the same capture, which is what weak bits '
|
|
343
|
+
f'look like. More reads may not settle them.')
|
|
344
|
+
else:
|
|
345
|
+
unchecked = sum(1 for r in result.resolutions if not r.verified)
|
|
346
|
+
if unchecked:
|
|
347
|
+
out += ['', f'Nothing left to re-read, but {unchecked} of these '
|
|
348
|
+
f'sectors came from inputs that carry no check value, '
|
|
349
|
+
f'so nothing confirmed them.']
|
|
350
|
+
else:
|
|
351
|
+
out += ['', 'Every sector confirmed by CRC. Nothing left to '
|
|
352
|
+
're-read.']
|
|
353
|
+
return '\n'.join(out)
|
diskstack/stack.py
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"""Decide, sector by sector, which read attempt wins.
|
|
2
|
+
|
|
3
|
+
Attempts are grouped by ``(cylinder, head, sector id)`` read out of the
|
|
4
|
+
decoded ID address mark -- never by position in the flux. Greaseweazle's
|
|
5
|
+
in-codec merge pairs sectors up when their start offsets are within 1000
|
|
6
|
+
bitcells (``codec/ibm/ibm.py``), which works across revolutions of one capture
|
|
7
|
+
and cannot work across two captures, where index alignment and motor speed
|
|
8
|
+
both differ.
|
|
9
|
+
|
|
10
|
+
Three tiers per sector:
|
|
11
|
+
|
|
12
|
+
1. any attempt whose own CRC or checksum passes wins outright;
|
|
13
|
+
2. otherwise the attempts are recombined into candidate payloads -- a byte-wise
|
|
14
|
+
majority, then one vote per input file, then each read's own bytes -- and the
|
|
15
|
+
first payload that satisfies a check value off the disk wins;
|
|
16
|
+
3. otherwise the largest cluster of attempts that agree exactly is emitted and
|
|
17
|
+
the sector is reported unresolved.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from collections import Counter, defaultdict
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Dict, Iterable, Iterator, List, Sequence, Tuple
|
|
26
|
+
|
|
27
|
+
from diskstack.candidates import Candidate, verify_sector
|
|
28
|
+
from diskstack.filler import make_filler
|
|
29
|
+
|
|
30
|
+
CLEAN = 'clean'
|
|
31
|
+
VOTED = 'recovered_by_vote'
|
|
32
|
+
UNRESOLVED = 'unresolved'
|
|
33
|
+
MISSING = 'missing'
|
|
34
|
+
|
|
35
|
+
STATUSES = (CLEAN, VOTED, UNRESOLVED, MISSING)
|
|
36
|
+
|
|
37
|
+
# How a tier 2 payload was put back together, best evidence first.
|
|
38
|
+
METHODS = ('majority', 'per_source_majority', 'cross_check')
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class Contribution:
|
|
43
|
+
"""One read attempt that produced the bytes diskstack chose."""
|
|
44
|
+
|
|
45
|
+
source: Path
|
|
46
|
+
rev: int
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class Resolution:
|
|
51
|
+
"""The outcome for one sector of the disk."""
|
|
52
|
+
|
|
53
|
+
cyl: int
|
|
54
|
+
head: int
|
|
55
|
+
sec_id: int
|
|
56
|
+
size: int
|
|
57
|
+
data: bytes
|
|
58
|
+
status: str
|
|
59
|
+
attempts: int = 0
|
|
60
|
+
good: int = 0
|
|
61
|
+
agreement: int = 0
|
|
62
|
+
sources: List[Contribution] = field(default_factory=list)
|
|
63
|
+
discarded: int = 0
|
|
64
|
+
unstable: bool = False
|
|
65
|
+
method: str = ''
|
|
66
|
+
verified: bool = False
|
|
67
|
+
contested: bool = False
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def key(self) -> Tuple[int, int, int]:
|
|
71
|
+
return (self.cyl, self.head, self.sec_id)
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def resolved(self) -> bool:
|
|
75
|
+
return self.status in (CLEAN, VOTED)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass
|
|
79
|
+
class StackResult:
|
|
80
|
+
"""Everything the merge decided, ready for writing out and reporting."""
|
|
81
|
+
|
|
82
|
+
resolutions: List[Resolution]
|
|
83
|
+
ranking: List[Path]
|
|
84
|
+
supplied: Dict[Path, int] = field(default_factory=dict)
|
|
85
|
+
sole_source: Dict[Path, int] = field(default_factory=dict)
|
|
86
|
+
|
|
87
|
+
def counts(self) -> Dict[str, int]:
|
|
88
|
+
counts = {s: 0 for s in STATUSES}
|
|
89
|
+
for res in self.resolutions:
|
|
90
|
+
counts[res.status] += 1
|
|
91
|
+
return counts
|
|
92
|
+
|
|
93
|
+
def by_track(self) -> Dict[Tuple[int, int], List[Resolution]]:
|
|
94
|
+
tracks: Dict[Tuple[int, int], List[Resolution]] = defaultdict(list)
|
|
95
|
+
for res in self.resolutions:
|
|
96
|
+
tracks[res.cyl, res.head].append(res)
|
|
97
|
+
return dict(tracks)
|
|
98
|
+
|
|
99
|
+
def sector_data(self) -> Dict[Tuple[int, int], Dict[int, bytes]]:
|
|
100
|
+
out: Dict[Tuple[int, int], Dict[int, bytes]] = defaultdict(dict)
|
|
101
|
+
for res in self.resolutions:
|
|
102
|
+
out[res.cyl, res.head][res.sec_id] = res.data
|
|
103
|
+
return dict(out)
|
|
104
|
+
|
|
105
|
+
def bad_sectors(self) -> Dict[Tuple[int, int], List[int]]:
|
|
106
|
+
out: Dict[Tuple[int, int], List[int]] = defaultdict(list)
|
|
107
|
+
for res in self.resolutions:
|
|
108
|
+
if not res.resolved:
|
|
109
|
+
out[res.cyl, res.head].append(res.sec_id)
|
|
110
|
+
return dict(out)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def rank_sources(candidates: Iterable[Candidate],
|
|
114
|
+
order: Sequence[Path]) -> List[Path]:
|
|
115
|
+
"""Input files, best first, by how many distinct sectors each read cleanly.
|
|
116
|
+
|
|
117
|
+
This is the tie-break for the byte vote: when two byte values are equally
|
|
118
|
+
popular, the one from the dump that read the rest of the disk best wins.
|
|
119
|
+
"""
|
|
120
|
+
good: Dict[Path, set] = {p: set() for p in order}
|
|
121
|
+
for cand in candidates:
|
|
122
|
+
if cand.data_crc_ok:
|
|
123
|
+
good.setdefault(cand.source, set()).add(cand.key)
|
|
124
|
+
position = {p: i for i, p in enumerate(order)}
|
|
125
|
+
return sorted(good, key=lambda p: (-len(good[p]), position.get(p, 0)))
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def largest_cluster(values: Sequence[bytes]) -> Tuple[bytes, int]:
|
|
129
|
+
"""Largest set of byte-identical payloads, ties to the earliest value.
|
|
130
|
+
|
|
131
|
+
``values`` must already be ordered best source first.
|
|
132
|
+
"""
|
|
133
|
+
counts = Counter(values)
|
|
134
|
+
best = max(counts.values())
|
|
135
|
+
tied = {value for value, n in counts.items() if n == best}
|
|
136
|
+
if len(tied) == 1:
|
|
137
|
+
return tied.pop(), best
|
|
138
|
+
return next(v for v in values if v in tied), best
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def majority_bytes(values: Sequence[bytes]) -> bytes:
|
|
142
|
+
"""Byte-wise majority over equal-length payloads.
|
|
143
|
+
|
|
144
|
+
Per byte position the most common value wins; a tie goes to the earliest
|
|
145
|
+
payload holding one of the tied values, so ``values`` must already be
|
|
146
|
+
ordered best source first.
|
|
147
|
+
"""
|
|
148
|
+
out = bytearray()
|
|
149
|
+
for column in zip(*values, strict=True):
|
|
150
|
+
if len(set(column)) == 1:
|
|
151
|
+
out.append(column[0])
|
|
152
|
+
continue
|
|
153
|
+
counts = Counter(column)
|
|
154
|
+
best = max(counts.values())
|
|
155
|
+
winners = {value for value, n in counts.items() if n == best}
|
|
156
|
+
if len(winners) == 1:
|
|
157
|
+
out.append(winners.pop())
|
|
158
|
+
else:
|
|
159
|
+
out.append(next(v for v in column if v in winners))
|
|
160
|
+
return bytes(out)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _check_trials(group: Sequence[Candidate]) -> List[Tuple[int, bytes]]:
|
|
164
|
+
"""(mark, check) pairs a voted payload can be tested against.
|
|
165
|
+
|
|
166
|
+
The check bytes come off the same damaged track as the payload, so the
|
|
167
|
+
vote covers them too and the voted value is tried alongside the ones that
|
|
168
|
+
were actually read.
|
|
169
|
+
"""
|
|
170
|
+
trials: List[Tuple[int, bytes]] = []
|
|
171
|
+
seen = set()
|
|
172
|
+
for cand in group:
|
|
173
|
+
if cand.check and (cand.mark, cand.check) not in seen:
|
|
174
|
+
seen.add((cand.mark, cand.check))
|
|
175
|
+
trials.append((cand.mark, cand.check))
|
|
176
|
+
with_check = [c for c in group if c.check]
|
|
177
|
+
if len(with_check) > 1:
|
|
178
|
+
size = Counter(len(c.check) for c in with_check).most_common(1)[0][0]
|
|
179
|
+
same = [c for c in with_check if len(c.check) == size]
|
|
180
|
+
voted = (Counter(c.mark for c in same).most_common(1)[0][0],
|
|
181
|
+
majority_bytes([c.check for c in same]))
|
|
182
|
+
if voted not in seen:
|
|
183
|
+
trials.append(voted)
|
|
184
|
+
return trials
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _per_source(group: Sequence[Candidate]) -> List[bytes]:
|
|
188
|
+
"""One payload per input file, best source first."""
|
|
189
|
+
by_source: Dict[Path, List[bytes]] = {}
|
|
190
|
+
for cand in group:
|
|
191
|
+
by_source.setdefault(cand.source, []).append(cand.data)
|
|
192
|
+
return [majority_bytes(v) for v in by_source.values()]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def reconstructions(group: Sequence[Candidate]) -> Iterator[Tuple[str, bytes]]:
|
|
196
|
+
"""Payloads worth testing against the check values that came off the disk.
|
|
197
|
+
|
|
198
|
+
The plain majority is what a vote means and comes first. The other two
|
|
199
|
+
cover what it cannot express: a capture with more revolutions than the
|
|
200
|
+
rest drowning out the sources that read the sector correctly, and a read
|
|
201
|
+
whose payload is fine but whose own check bytes were the damaged part.
|
|
202
|
+
"""
|
|
203
|
+
payloads = [c.data for c in group]
|
|
204
|
+
plain, per_source = METHODS[:2]
|
|
205
|
+
seen = set()
|
|
206
|
+
for method, data in [(plain, majority_bytes(payloads)),
|
|
207
|
+
(per_source, majority_bytes(_per_source(group)))]:
|
|
208
|
+
if data not in seen:
|
|
209
|
+
seen.add(data)
|
|
210
|
+
yield method, data
|
|
211
|
+
for data in payloads:
|
|
212
|
+
if data not in seen:
|
|
213
|
+
seen.add(data)
|
|
214
|
+
yield METHODS[2], data
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def unstable(group: Sequence[Candidate]) -> bool:
|
|
218
|
+
"""True if some input read this sector two different ways on its own.
|
|
219
|
+
|
|
220
|
+
Two revolutions of one capture disagreeing is the signature of weak bits,
|
|
221
|
+
which are deliberate on protected disks and do not settle down however
|
|
222
|
+
many more times the disk is read. Comparing revolution against revolution
|
|
223
|
+
rather than pooling the whole file keeps a repeated ``--pll`` out of it:
|
|
224
|
+
two settings that read the same revolution two ways say something about
|
|
225
|
+
the decode, not about the disk.
|
|
226
|
+
"""
|
|
227
|
+
per_source: Dict[Path, Dict[int, set]] = defaultdict(dict)
|
|
228
|
+
for cand in group:
|
|
229
|
+
per_source[cand.source].setdefault(cand.rev, set()).add(cand.data)
|
|
230
|
+
for by_rev in per_source.values():
|
|
231
|
+
revs = list(by_rev.values())
|
|
232
|
+
if len(revs) > 1 and set.union(*revs) != set.intersection(*revs):
|
|
233
|
+
return True
|
|
234
|
+
return False
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def resolve(key: Tuple[int, int, int], size: int,
|
|
238
|
+
group: Sequence[Candidate], vote: bool = True) -> Resolution:
|
|
239
|
+
"""Apply the three tiers to one sector's read attempts."""
|
|
240
|
+
cyl, head, sec_id = key
|
|
241
|
+
usable = [c for c in group if len(c.data) == size]
|
|
242
|
+
res = Resolution(cyl=cyl, head=head, sec_id=sec_id, size=size,
|
|
243
|
+
data=make_filler(size), status=MISSING,
|
|
244
|
+
attempts=len(group), discarded=len(group) - len(usable))
|
|
245
|
+
if not usable:
|
|
246
|
+
return res
|
|
247
|
+
res.good = sum(1 for c in usable if c.data_crc_ok)
|
|
248
|
+
res.unstable = unstable(usable)
|
|
249
|
+
|
|
250
|
+
clean = [c for c in usable if c.data_crc_ok]
|
|
251
|
+
if clean:
|
|
252
|
+
# A read that satisfied a check value off the disk outranks one that
|
|
253
|
+
# was merely never contradicted, which is all a sector image can say.
|
|
254
|
+
verified = [c for c in clean if c.check]
|
|
255
|
+
data, _ = largest_cluster([c.data for c in verified or clean])
|
|
256
|
+
matched = [c for c in clean if c.data == data]
|
|
257
|
+
res.data, res.status = data, CLEAN
|
|
258
|
+
res.agreement = len(matched)
|
|
259
|
+
res.verified = any(c.check for c in matched)
|
|
260
|
+
res.contested = len(matched) < len(clean)
|
|
261
|
+
res.sources = [Contribution(c.source, c.rev) for c in matched]
|
|
262
|
+
return res
|
|
263
|
+
|
|
264
|
+
if vote and len(usable) > 1:
|
|
265
|
+
codec = usable[0].codec
|
|
266
|
+
trials = _check_trials(usable)
|
|
267
|
+
for method, data in reconstructions(usable):
|
|
268
|
+
if any(verify_sector(codec, mark, data, check)
|
|
269
|
+
for mark, check in trials):
|
|
270
|
+
res.data, res.status, res.method = data, VOTED, method
|
|
271
|
+
res.verified = True
|
|
272
|
+
matched = [c for c in usable if c.data == data]
|
|
273
|
+
res.agreement = len(matched)
|
|
274
|
+
res.sources = [Contribution(c.source, c.rev)
|
|
275
|
+
for c in matched or usable]
|
|
276
|
+
return res
|
|
277
|
+
|
|
278
|
+
data, agree = largest_cluster([c.data for c in usable])
|
|
279
|
+
res.data, res.status, res.agreement = data, UNRESOLVED, agree
|
|
280
|
+
res.sources = [Contribution(c.source, c.rev) for c in usable
|
|
281
|
+
if c.data == data]
|
|
282
|
+
return res
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def stack(candidates: Iterable[Candidate],
|
|
286
|
+
expected: Sequence[Tuple[int, int, int]],
|
|
287
|
+
sizes: Dict[Tuple[int, int, int], int],
|
|
288
|
+
order: Sequence[Path],
|
|
289
|
+
vote: bool = True) -> StackResult:
|
|
290
|
+
"""Merge every read attempt into one resolution per expected sector."""
|
|
291
|
+
candidates = list(candidates)
|
|
292
|
+
ranking = rank_sources(candidates, order)
|
|
293
|
+
rank = {path: i for i, path in enumerate(ranking)}
|
|
294
|
+
|
|
295
|
+
groups: Dict[Tuple[int, int, int], List[Candidate]] = defaultdict(list)
|
|
296
|
+
for cand in candidates:
|
|
297
|
+
groups[cand.key].append(cand)
|
|
298
|
+
for group in groups.values():
|
|
299
|
+
group.sort(key=lambda c: (rank.get(c.source, len(rank)), c.rev))
|
|
300
|
+
|
|
301
|
+
resolutions = [resolve(key, sizes[key], groups.get(key, ()), vote=vote)
|
|
302
|
+
for key in expected]
|
|
303
|
+
|
|
304
|
+
supplied = {path: 0 for path in order}
|
|
305
|
+
sole = {path: 0 for path in order}
|
|
306
|
+
for res in resolutions:
|
|
307
|
+
if not res.resolved:
|
|
308
|
+
continue
|
|
309
|
+
paths = {c.source for c in res.sources}
|
|
310
|
+
for path in paths:
|
|
311
|
+
supplied[path] = supplied.get(path, 0) + 1
|
|
312
|
+
if len(paths) == 1:
|
|
313
|
+
path = next(iter(paths))
|
|
314
|
+
sole[path] = sole.get(path, 0) + 1
|
|
315
|
+
|
|
316
|
+
return StackResult(resolutions=resolutions, ranking=ranking,
|
|
317
|
+
supplied=supplied, sole_source=sole)
|