diskstack 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. diskstack/__init__.py +5 -0
  2. diskstack/_vendor/__init__.py +0 -0
  3. diskstack/_vendor/greaseweazle/COPYING +24 -0
  4. diskstack/_vendor/greaseweazle/__init__.py +2 -0
  5. diskstack/_vendor/greaseweazle/codec/__init__.py +0 -0
  6. diskstack/_vendor/greaseweazle/codec/amiga/__init__.py +0 -0
  7. diskstack/_vendor/greaseweazle/codec/amiga/amigados.py +221 -0
  8. diskstack/_vendor/greaseweazle/codec/apple2/__init__.py +0 -0
  9. diskstack/_vendor/greaseweazle/codec/apple2/apple2_gcr.py +223 -0
  10. diskstack/_vendor/greaseweazle/codec/bitcell.py +117 -0
  11. diskstack/_vendor/greaseweazle/codec/codec.py +362 -0
  12. diskstack/_vendor/greaseweazle/codec/commodore/__init__.py +0 -0
  13. diskstack/_vendor/greaseweazle/codec/commodore/c64_gcr.py +229 -0
  14. diskstack/_vendor/greaseweazle/codec/datageneral/__init__.py +0 -0
  15. diskstack/_vendor/greaseweazle/codec/datageneral/datageneral.py +211 -0
  16. diskstack/_vendor/greaseweazle/codec/hp/__init__.py +0 -0
  17. diskstack/_vendor/greaseweazle/codec/hp/hp_mmfm.py +258 -0
  18. diskstack/_vendor/greaseweazle/codec/ibm/__init__.py +0 -0
  19. diskstack/_vendor/greaseweazle/codec/ibm/ibm.py +1005 -0
  20. diskstack/_vendor/greaseweazle/codec/macintosh/__init__.py +0 -0
  21. diskstack/_vendor/greaseweazle/codec/macintosh/mac_gcr.py +238 -0
  22. diskstack/_vendor/greaseweazle/codec/micropolis/__init__.py +0 -0
  23. diskstack/_vendor/greaseweazle/codec/micropolis/micropolis.py +233 -0
  24. diskstack/_vendor/greaseweazle/codec/northstar/__init__.py +0 -0
  25. diskstack/_vendor/greaseweazle/codec/northstar/northstar.py +210 -0
  26. diskstack/_vendor/greaseweazle/data/__init__.py +0 -0
  27. diskstack/_vendor/greaseweazle/data/diskdefs.cfg +81 -0
  28. diskstack/_vendor/greaseweazle/data/diskdefs_acorn.cfg +119 -0
  29. diskstack/_vendor/greaseweazle/data/diskdefs_akai.cfg +27 -0
  30. diskstack/_vendor/greaseweazle/data/diskdefs_amiga.cfg +17 -0
  31. diskstack/_vendor/greaseweazle/data/diskdefs_apple2.cfg +31 -0
  32. diskstack/_vendor/greaseweazle/data/diskdefs_apricot.cfg +25 -0
  33. diskstack/_vendor/greaseweazle/data/diskdefs_atari.cfg +33 -0
  34. diskstack/_vendor/greaseweazle/data/diskdefs_atarist.cfg +76 -0
  35. diskstack/_vendor/greaseweazle/data/diskdefs_coco.cfg +98 -0
  36. diskstack/_vendor/greaseweazle/data/diskdefs_commodore.cfg +127 -0
  37. diskstack/_vendor/greaseweazle/data/diskdefs_datageneral.cfg +8 -0
  38. diskstack/_vendor/greaseweazle/data/diskdefs_dec.cfg +37 -0
  39. diskstack/_vendor/greaseweazle/data/diskdefs_dragon.cfg +61 -0
  40. diskstack/_vendor/greaseweazle/data/diskdefs_eagle.cfg +25 -0
  41. diskstack/_vendor/greaseweazle/data/diskdefs_ensoniq.cfg +39 -0
  42. diskstack/_vendor/greaseweazle/data/diskdefs_epson.cfg +95 -0
  43. diskstack/_vendor/greaseweazle/data/diskdefs_gem.cfg +15 -0
  44. diskstack/_vendor/greaseweazle/data/diskdefs_hp.cfg +25 -0
  45. diskstack/_vendor/greaseweazle/data/diskdefs_ibm.cfg +140 -0
  46. diskstack/_vendor/greaseweazle/data/diskdefs_kaypro.cfg +58 -0
  47. diskstack/_vendor/greaseweazle/data/diskdefs_luxor.cfg +96 -0
  48. diskstack/_vendor/greaseweazle/data/diskdefs_mac.cfg +71 -0
  49. diskstack/_vendor/greaseweazle/data/diskdefs_micropolis.cfg +73 -0
  50. diskstack/_vendor/greaseweazle/data/diskdefs_mm1.cfg +69 -0
  51. diskstack/_vendor/greaseweazle/data/diskdefs_msx.cfg +45 -0
  52. diskstack/_vendor/greaseweazle/data/diskdefs_northstar.cfg +37 -0
  53. diskstack/_vendor/greaseweazle/data/diskdefs_occ1.cfg +23 -0
  54. diskstack/_vendor/greaseweazle/data/diskdefs_olivetti.cfg +18 -0
  55. diskstack/_vendor/greaseweazle/data/diskdefs_pc98.cfg +69 -0
  56. diskstack/_vendor/greaseweazle/data/diskdefs_raw.cfg +25 -0
  57. diskstack/_vendor/greaseweazle/data/diskdefs_rm.cfg +98 -0
  58. diskstack/_vendor/greaseweazle/data/diskdefs_sci.cfg +15 -0
  59. diskstack/_vendor/greaseweazle/data/diskdefs_sega.cfg +13 -0
  60. diskstack/_vendor/greaseweazle/data/diskdefs_sharp.cfg +13 -0
  61. diskstack/_vendor/greaseweazle/data/diskdefs_thomson.cfg +78 -0
  62. diskstack/_vendor/greaseweazle/data/diskdefs_tsc.cfg +81 -0
  63. diskstack/_vendor/greaseweazle/data/diskdefs_xerox.cfg +59 -0
  64. diskstack/_vendor/greaseweazle/data/diskdefs_zx.cfg +245 -0
  65. diskstack/_vendor/greaseweazle/error.py +19 -0
  66. diskstack/_vendor/greaseweazle/flux.py +315 -0
  67. diskstack/_vendor/greaseweazle/image/__init__.py +0 -0
  68. diskstack/_vendor/greaseweazle/image/a2r.py +133 -0
  69. diskstack/_vendor/greaseweazle/image/acorn.py +28 -0
  70. diskstack/_vendor/greaseweazle/image/adf.py +15 -0
  71. diskstack/_vendor/greaseweazle/image/apple2.py +18 -0
  72. diskstack/_vendor/greaseweazle/image/apridisk.py +107 -0
  73. diskstack/_vendor/greaseweazle/image/caps.py +464 -0
  74. diskstack/_vendor/greaseweazle/image/d64.py +51 -0
  75. diskstack/_vendor/greaseweazle/image/d81.py +28 -0
  76. diskstack/_vendor/greaseweazle/image/d88.py +211 -0
  77. diskstack/_vendor/greaseweazle/image/dcp.py +45 -0
  78. diskstack/_vendor/greaseweazle/image/dim.py +53 -0
  79. diskstack/_vendor/greaseweazle/image/dmk.py +166 -0
  80. diskstack/_vendor/greaseweazle/image/dsk.py +31 -0
  81. diskstack/_vendor/greaseweazle/image/edsk.py +591 -0
  82. diskstack/_vendor/greaseweazle/image/fd.py +23 -0
  83. diskstack/_vendor/greaseweazle/image/fdi.py +41 -0
  84. diskstack/_vendor/greaseweazle/image/hdm.py +15 -0
  85. diskstack/_vendor/greaseweazle/image/hfe.py +742 -0
  86. diskstack/_vendor/greaseweazle/image/image.py +130 -0
  87. diskstack/_vendor/greaseweazle/image/imd.py +201 -0
  88. diskstack/_vendor/greaseweazle/image/img.py +111 -0
  89. diskstack/_vendor/greaseweazle/image/kryoflux.py +304 -0
  90. diskstack/_vendor/greaseweazle/image/mgt.py +15 -0
  91. diskstack/_vendor/greaseweazle/image/msa.py +153 -0
  92. diskstack/_vendor/greaseweazle/image/nfd.py +113 -0
  93. diskstack/_vendor/greaseweazle/image/nsi.py +47 -0
  94. diskstack/_vendor/greaseweazle/image/scp.py +471 -0
  95. diskstack/_vendor/greaseweazle/image/sf7.py +15 -0
  96. diskstack/_vendor/greaseweazle/image/sharp2d.py +15 -0
  97. diskstack/_vendor/greaseweazle/image/td0.py +141 -0
  98. diskstack/_vendor/greaseweazle/image/xdf.py +15 -0
  99. diskstack/_vendor/greaseweazle/optimised/__init__.py +22 -0
  100. diskstack/_vendor/greaseweazle/tools/__init__.py +0 -0
  101. diskstack/_vendor/greaseweazle/tools/util.py +133 -0
  102. diskstack/_vendor/greaseweazle/track.py +494 -0
  103. diskstack/cache.py +210 -0
  104. diskstack/candidates.py +396 -0
  105. diskstack/cli.py +348 -0
  106. diskstack/errors.py +5 -0
  107. diskstack/filler.py +29 -0
  108. diskstack/formats.py +381 -0
  109. diskstack/parallel.py +73 -0
  110. diskstack/report.py +353 -0
  111. diskstack/stack.py +317 -0
  112. diskstack-1.0.0.dist-info/METADATA +329 -0
  113. diskstack-1.0.0.dist-info/RECORD +119 -0
  114. diskstack-1.0.0.dist-info/WHEEL +5 -0
  115. diskstack-1.0.0.dist-info/entry_points.txt +2 -0
  116. diskstack-1.0.0.dist-info/licenses/LICENSE +21 -0
  117. diskstack-1.0.0.dist-info/licenses/NOTICE +55 -0
  118. diskstack-1.0.0.dist-info/licenses/diskstack/_vendor/greaseweazle/COPYING +24 -0
  119. diskstack-1.0.0.dist-info/top_level.txt +1 -0
diskstack/report.py ADDED
@@ -0,0 +1,353 @@
1
+ """The JSON report, the stdout tables, and the re-read command.
2
+
3
+ The re-read line is the point of the tool: it names only the tracks that are
4
+ still bad, so the next pass over the disk is seconds of drive time instead of
5
+ minutes, and the result goes straight back in as another input.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import datetime
11
+ import json
12
+ from collections import defaultdict
13
+ from dataclasses import asdict, dataclass
14
+ from pathlib import Path
15
+ from typing import Dict, List, Optional, Sequence
16
+
17
+ from diskstack import __version__
18
+ from diskstack.candidates import SourceInfo
19
+ from diskstack.stack import CLEAN, MISSING, UNRESOLVED, VOTED, StackResult
20
+
21
+ SCHEMA_VERSION = 1
22
+
23
+
24
+ @dataclass
25
+ class Progress:
26
+ """How a merge moved against the report from the run before it."""
27
+
28
+ recovered: int
29
+ lost: int
30
+ still_bad: int
31
+
32
+
33
+ def read_previous(path: Path) -> Optional[dict]:
34
+ """The report already sitting at ``path``, if it is one diskstack wrote."""
35
+ try:
36
+ previous = json.loads(path.read_text(encoding='utf-8'))
37
+ except (OSError, ValueError):
38
+ return None
39
+ return previous if isinstance(previous, dict) else None
40
+
41
+
42
+ def compare(previous: dict, result: StackResult) -> Optional[Progress]:
43
+ """What this merge changed since ``previous``, or None if not comparable.
44
+
45
+ The two runs have to cover the same sectors. Otherwise the report is of
46
+ another disk and a count of what changed would mean nothing.
47
+ """
48
+ if previous.get('schema') != SCHEMA_VERSION:
49
+ return None
50
+ try:
51
+ was = {(s['cyl'], s['head'], s['sec_id']):
52
+ s['status'] in (CLEAN, VOTED)
53
+ for s in previous['sectors']}
54
+ except (KeyError, TypeError):
55
+ return None
56
+ now = {res.key: res.resolved for res in result.resolutions}
57
+ if set(was) != set(now):
58
+ return None
59
+ return Progress(
60
+ recovered=sum(1 for k, ok in now.items() if ok and not was[k]),
61
+ lost=sum(1 for k, ok in now.items() if not ok and was[k]),
62
+ still_bad=sum(1 for k, ok in now.items() if not ok and not was[k]))
63
+
64
+
65
+ def progress_note(progress: Optional[Progress]) -> List[str]:
66
+ """The one line that says whether the last re-read was worth doing."""
67
+ if progress is None:
68
+ return []
69
+ if progress.recovered or progress.lost:
70
+ parts = [f'{progress.recovered} recovered']
71
+ if progress.lost:
72
+ parts.append(f'{progress.lost} lost')
73
+ return [f'Since the last report: {", ".join(parts)}.']
74
+ if progress.still_bad:
75
+ return [f'No change since the last report: the same '
76
+ f'{progress.still_bad} sectors are still bad. Another pass at '
77
+ f'the same settings may not do better; --pll is the other '
78
+ f'thing to vary.']
79
+ return []
80
+
81
+
82
+ def _ranges(values: Sequence[int]) -> str:
83
+ """Collapse sorted integers into Greaseweazle range syntax: 3,7-9."""
84
+ parts, start, prev = [], None, None
85
+ for value in values:
86
+ if start is None:
87
+ start = prev = value
88
+ elif value == prev + 1:
89
+ prev = value
90
+ else:
91
+ parts.append(str(start) if start == prev else f'{start}-{prev}')
92
+ start = prev = value
93
+ if start is not None:
94
+ parts.append(str(start) if start == prev else f'{start}-{prev}')
95
+ return ','.join(parts)
96
+
97
+
98
+ def reread_trackspecs(result: StackResult) -> List[str]:
99
+ """Greaseweazle ``--tracks=`` specs covering every unresolved sector.
100
+
101
+ Heads whose bad cylinders are identical share one spec; otherwise each
102
+ head gets its own, so the user never re-reads a track that is already good.
103
+ """
104
+ per_head: Dict[int, set] = defaultdict(set)
105
+ for res in result.resolutions:
106
+ if not res.resolved:
107
+ per_head[res.head].add(res.cyl)
108
+ grouped: Dict[frozenset, List[int]] = defaultdict(list)
109
+ for head, cyls in per_head.items():
110
+ grouped[frozenset(cyls)].append(head)
111
+ specs = []
112
+ for cyls, heads in sorted(grouped.items(),
113
+ key=lambda kv: (sorted(kv[1]), sorted(kv[0]))):
114
+ specs.append(f'c={_ranges(sorted(cyls))}:h={_ranges(sorted(heads))}')
115
+ return specs
116
+
117
+
118
+ def reread_commands(result: StackResult, name: str = 'retry.scp') -> List[str]:
119
+ """Full ``gw read`` command lines for the tracks still missing data."""
120
+ specs = reread_trackspecs(result)
121
+ if len(specs) == 1:
122
+ return [f'gw read {name} --tracks={specs[0]}']
123
+ stem, suffix = name.rsplit('.', 1) if '.' in name else (name, 'scp')
124
+ return [f'gw read {stem}{i}.{suffix} --tracks={spec}'
125
+ for i, spec in enumerate(specs, 1)]
126
+
127
+
128
+ def summary_line(result: StackResult) -> str:
129
+ """The headline: how much of the disk came out readable, and from where."""
130
+ counts = result.counts()
131
+ total = len(result.resolutions)
132
+ files = len({path for path, n in result.supplied.items() if n})
133
+ parts = [f'{counts[CLEAN]} clean '
134
+ f'(from {files} file{"s" if files != 1 else ""})']
135
+ if counts[VOTED]:
136
+ parts.append(f'{counts[VOTED]} recovered by vote')
137
+ if counts[UNRESOLVED]:
138
+ parts.append(f'{counts[UNRESOLVED]} unresolved')
139
+ if counts[MISSING]:
140
+ parts.append(f'{counts[MISSING]} missing')
141
+ return f'{total} sectors: ' + ', '.join(parts)
142
+
143
+
144
+ def render_table(headers: Sequence[str],
145
+ rows: Sequence[Sequence[object]],
146
+ align: str = '') -> str:
147
+ """Fixed-width table; ``align`` is one of 'l'/'r' per column."""
148
+ cells = [[str(c) for c in row] for row in rows]
149
+ widths = [max(len(h), *(len(r[i]) for r in cells)) if cells else len(h)
150
+ for i, h in enumerate(headers)]
151
+ align = (align + 'l' * len(headers))[:len(headers)]
152
+
153
+ def line(values):
154
+ out = []
155
+ for value, width, how in zip(values, widths, align, strict=True):
156
+ out.append(value.rjust(width) if how == 'r' else value.ljust(width))
157
+ return ' ' + ' '.join(out).rstrip()
158
+
159
+ return '\n'.join([line(headers)] + [line(r) for r in cells])
160
+
161
+
162
+ def source_table(sources: Sequence[SourceInfo], result: StackResult) -> str:
163
+ """Per-input-file provenance: what each dump brought to the merge."""
164
+ rows = []
165
+ for info in sources:
166
+ rows.append([
167
+ info.path.name, info.kind, info.revolutions, info.candidates,
168
+ info.good, result.supplied.get(info.path, 0),
169
+ result.sole_source.get(info.path, 0),
170
+ ])
171
+ return render_table(
172
+ ['file', 'type', 'revs', 'attempts', 'good', 'supplied', 'sole'],
173
+ rows, align='llrrrrr')
174
+
175
+
176
+ def _track_rows(result: StackResult) -> List[List[object]]:
177
+ rows = []
178
+ for (cyl, head), sectors in sorted(result.by_track().items()):
179
+ counts = {CLEAN: 0, VOTED: 0, UNRESOLVED: 0, MISSING: 0}
180
+ for res in sectors:
181
+ counts[res.status] += 1
182
+ if not counts[UNRESOLVED] and not counts[MISSING]:
183
+ continue
184
+ bad = sorted(r.sec_id for r in sectors if not r.resolved)
185
+ rows.append([cyl, head, counts[CLEAN], counts[VOTED],
186
+ counts[UNRESOLVED], counts[MISSING],
187
+ ','.join(str(s) for s in bad)])
188
+ return rows
189
+
190
+
191
+ def track_table(result: StackResult) -> str:
192
+ """Tracks that still hold a sector diskstack could not confirm."""
193
+ return render_table(
194
+ ['cyl', 'head', 'clean', 'voted', 'unresolved', 'missing',
195
+ 'sector ids still bad'],
196
+ _track_rows(result), align='rrrrrrl')
197
+
198
+
199
+ def build(result: StackResult, sources: Sequence[SourceInfo],
200
+ fmt_name: str, fmt_detail: str, output: Path,
201
+ retry_name: str = 'retry.scp',
202
+ progress: Optional[Progress] = None) -> dict:
203
+ """The machine-readable report: every sector, with its provenance."""
204
+ counts = result.counts()
205
+ return {
206
+ 'schema': SCHEMA_VERSION,
207
+ 'tool': 'diskstack',
208
+ 'version': __version__,
209
+ 'generated': datetime.datetime.now(datetime.timezone.utc)
210
+ .replace(microsecond=0).isoformat(),
211
+ 'format': fmt_name,
212
+ 'format_detail': fmt_detail,
213
+ 'output': str(output),
214
+ 'inputs': [
215
+ {
216
+ 'path': str(info.path),
217
+ 'type': info.kind,
218
+ 'revolutions': info.revolutions,
219
+ 'attempts': info.candidates,
220
+ 'good': info.good,
221
+ 'supplied': result.supplied.get(info.path, 0),
222
+ 'sole_source': result.sole_source.get(info.path, 0),
223
+ 'unexpected_sectors': len(info.unexpected),
224
+ 'bytes': info.size,
225
+ 'expected_bytes': info.expected_size or None,
226
+ 'cached': info.cached,
227
+ }
228
+ for info in sources
229
+ ],
230
+ 'totals': {
231
+ 'sectors': len(result.resolutions),
232
+ 'clean': counts[CLEAN],
233
+ 'recovered_by_vote': counts[VOTED],
234
+ 'unresolved': counts[UNRESOLVED],
235
+ 'missing': counts[MISSING],
236
+ 'verified': sum(1 for r in result.resolutions if r.verified),
237
+ 'contested': sum(1 for r in result.resolutions if r.contested),
238
+ },
239
+ 'sectors': [
240
+ {
241
+ 'cyl': res.cyl,
242
+ 'head': res.head,
243
+ 'sec_id': res.sec_id,
244
+ 'size': res.size,
245
+ 'status': res.status,
246
+ 'attempts': res.attempts,
247
+ 'good': res.good,
248
+ 'agreement': res.agreement,
249
+ 'discarded': res.discarded,
250
+ 'unstable': res.unstable,
251
+ 'contested': res.contested,
252
+ 'verified': res.verified,
253
+ 'method': res.method or None,
254
+ 'sources': [{'path': str(c.source), 'rev': c.rev}
255
+ for c in res.sources],
256
+ }
257
+ for res in result.resolutions
258
+ ],
259
+ 'reread': {
260
+ 'tracks': reread_trackspecs(result),
261
+ 'commands': reread_commands(result, retry_name),
262
+ },
263
+ 'since_last_report': asdict(progress) if progress else None,
264
+ }
265
+
266
+
267
+ def write(report: dict, path: Path) -> None:
268
+ path.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
269
+
270
+
271
+ def warnings(sources: Sequence[SourceInfo], fmt_name: str) -> List[str]:
272
+ """What the user should know about the inputs before trusting the merge."""
273
+ out = []
274
+ for info in sources:
275
+ if n := len(info.unexpected):
276
+ out.append(f'{info.path.name}: ignored {n} sector'
277
+ f'{"s" if n != 1 else ""} the format does not expect')
278
+ if info.wrong_size:
279
+ tail = ('the extra sectors were ignored'
280
+ if info.size > info.expected_size
281
+ else 'the missing tail counts as unread')
282
+ out.append(f'{info.path.name}: {info.size} bytes where {fmt_name} '
283
+ f'is {info.expected_size}, so {tail}. Pass --format if '
284
+ f'that is the wrong disk format.')
285
+ return out
286
+
287
+
288
+ def cache_note(sources: Sequence[SourceInfo]) -> List[str]:
289
+ """Say when a run reused a stored decode rather than reading the flux."""
290
+ reused = sum(1 for info in sources if info.cached)
291
+ if not reused:
292
+ return []
293
+ return [f'{reused} of {len(sources)} inputs came out of the cache instead '
294
+ f'of being decoded again. --no-cache turns that off.']
295
+
296
+
297
+ def contested_note(result: StackResult) -> List[str]:
298
+ """Sectors where the dumps disagreed and every reading looked good.
299
+
300
+ Nothing arbitrates this: the merge keeps the best-ranked dump's copy. It is
301
+ what deliberate weak bits look like, and what two dumps of two different
302
+ disks look like.
303
+ """
304
+ bad = [r for r in result.resolutions if r.contested]
305
+ if not bad:
306
+ return []
307
+ where = ', '.join(f'c{r.cyl}:h{r.head}:s{r.sec_id}' for r in bad[:4])
308
+ if len(bad) > 4:
309
+ where += f', and {len(bad) - 4} more'
310
+ return [f'{len(bad)} sector{"s" if len(bad) != 1 else ""} came out good '
311
+ f'in more than one dump but with different bytes ({where}). '
312
+ f"diskstack kept the best-ranked dump's copy and flagged them "
313
+ f'contested in the report.']
314
+
315
+
316
+ def render(result: StackResult, sources: Sequence[SourceInfo],
317
+ fmt_name: str, fmt_detail: str,
318
+ retry_name: str = 'retry.scp',
319
+ tables: bool = True,
320
+ progress: Optional[Progress] = None) -> str:
321
+ """The stdout view: summary, provenance, and what to re-read."""
322
+ out = ([f'Format: {fmt_name} ({fmt_detail})', '', summary_line(result)]
323
+ + progress_note(progress))
324
+ if tables:
325
+ out += ['', 'Sources', source_table(sources, result)]
326
+ rows = _track_rows(result)
327
+ if rows:
328
+ out += ['', 'Tracks needing attention', track_table(result)]
329
+ notes = (warnings(sources, fmt_name) + cache_note(sources)
330
+ + contested_note(result))
331
+ if notes:
332
+ out += [''] + notes
333
+ commands = reread_commands(result, retry_name)
334
+ if commands:
335
+ out += ['', 'Re-read just these tracks, then run diskstack again '
336
+ 'with the new capture added:']
337
+ out += [' ' + c for c in commands]
338
+ shaky = sum(1 for r in result.resolutions
339
+ if not r.resolved and r.unstable)
340
+ if shaky:
341
+ out.append(f'{shaky} of those sectors read differently on every '
342
+ f'pass of the same capture, which is what weak bits '
343
+ f'look like. More reads may not settle them.')
344
+ else:
345
+ unchecked = sum(1 for r in result.resolutions if not r.verified)
346
+ if unchecked:
347
+ out += ['', f'Nothing left to re-read, but {unchecked} of these '
348
+ f'sectors came from inputs that carry no check value, '
349
+ f'so nothing confirmed them.']
350
+ else:
351
+ out += ['', 'Every sector confirmed by CRC. Nothing left to '
352
+ 're-read.']
353
+ return '\n'.join(out)
diskstack/stack.py ADDED
@@ -0,0 +1,317 @@
1
+ """Decide, sector by sector, which read attempt wins.
2
+
3
+ Attempts are grouped by ``(cylinder, head, sector id)`` read out of the
4
+ decoded ID address mark -- never by position in the flux. Greaseweazle's
5
+ in-codec merge pairs sectors up when their start offsets are within 1000
6
+ bitcells (``codec/ibm/ibm.py``), which works across revolutions of one capture
7
+ and cannot work across two captures, where index alignment and motor speed
8
+ both differ.
9
+
10
+ Three tiers per sector:
11
+
12
+ 1. any attempt whose own CRC or checksum passes wins outright;
13
+ 2. otherwise the attempts are recombined into candidate payloads -- a byte-wise
14
+ majority, then one vote per input file, then each read's own bytes -- and the
15
+ first payload that satisfies a check value off the disk wins;
16
+ 3. otherwise the largest cluster of attempts that agree exactly is emitted and
17
+ the sector is reported unresolved.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ from collections import Counter, defaultdict
23
+ from dataclasses import dataclass, field
24
+ from pathlib import Path
25
+ from typing import Dict, Iterable, Iterator, List, Sequence, Tuple
26
+
27
+ from diskstack.candidates import Candidate, verify_sector
28
+ from diskstack.filler import make_filler
29
+
30
+ CLEAN = 'clean'
31
+ VOTED = 'recovered_by_vote'
32
+ UNRESOLVED = 'unresolved'
33
+ MISSING = 'missing'
34
+
35
+ STATUSES = (CLEAN, VOTED, UNRESOLVED, MISSING)
36
+
37
+ # How a tier 2 payload was put back together, best evidence first.
38
+ METHODS = ('majority', 'per_source_majority', 'cross_check')
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class Contribution:
43
+ """One read attempt that produced the bytes diskstack chose."""
44
+
45
+ source: Path
46
+ rev: int
47
+
48
+
49
+ @dataclass
50
+ class Resolution:
51
+ """The outcome for one sector of the disk."""
52
+
53
+ cyl: int
54
+ head: int
55
+ sec_id: int
56
+ size: int
57
+ data: bytes
58
+ status: str
59
+ attempts: int = 0
60
+ good: int = 0
61
+ agreement: int = 0
62
+ sources: List[Contribution] = field(default_factory=list)
63
+ discarded: int = 0
64
+ unstable: bool = False
65
+ method: str = ''
66
+ verified: bool = False
67
+ contested: bool = False
68
+
69
+ @property
70
+ def key(self) -> Tuple[int, int, int]:
71
+ return (self.cyl, self.head, self.sec_id)
72
+
73
+ @property
74
+ def resolved(self) -> bool:
75
+ return self.status in (CLEAN, VOTED)
76
+
77
+
78
+ @dataclass
79
+ class StackResult:
80
+ """Everything the merge decided, ready for writing out and reporting."""
81
+
82
+ resolutions: List[Resolution]
83
+ ranking: List[Path]
84
+ supplied: Dict[Path, int] = field(default_factory=dict)
85
+ sole_source: Dict[Path, int] = field(default_factory=dict)
86
+
87
+ def counts(self) -> Dict[str, int]:
88
+ counts = {s: 0 for s in STATUSES}
89
+ for res in self.resolutions:
90
+ counts[res.status] += 1
91
+ return counts
92
+
93
+ def by_track(self) -> Dict[Tuple[int, int], List[Resolution]]:
94
+ tracks: Dict[Tuple[int, int], List[Resolution]] = defaultdict(list)
95
+ for res in self.resolutions:
96
+ tracks[res.cyl, res.head].append(res)
97
+ return dict(tracks)
98
+
99
+ def sector_data(self) -> Dict[Tuple[int, int], Dict[int, bytes]]:
100
+ out: Dict[Tuple[int, int], Dict[int, bytes]] = defaultdict(dict)
101
+ for res in self.resolutions:
102
+ out[res.cyl, res.head][res.sec_id] = res.data
103
+ return dict(out)
104
+
105
+ def bad_sectors(self) -> Dict[Tuple[int, int], List[int]]:
106
+ out: Dict[Tuple[int, int], List[int]] = defaultdict(list)
107
+ for res in self.resolutions:
108
+ if not res.resolved:
109
+ out[res.cyl, res.head].append(res.sec_id)
110
+ return dict(out)
111
+
112
+
113
+ def rank_sources(candidates: Iterable[Candidate],
114
+ order: Sequence[Path]) -> List[Path]:
115
+ """Input files, best first, by how many distinct sectors each read cleanly.
116
+
117
+ This is the tie-break for the byte vote: when two byte values are equally
118
+ popular, the one from the dump that read the rest of the disk best wins.
119
+ """
120
+ good: Dict[Path, set] = {p: set() for p in order}
121
+ for cand in candidates:
122
+ if cand.data_crc_ok:
123
+ good.setdefault(cand.source, set()).add(cand.key)
124
+ position = {p: i for i, p in enumerate(order)}
125
+ return sorted(good, key=lambda p: (-len(good[p]), position.get(p, 0)))
126
+
127
+
128
+ def largest_cluster(values: Sequence[bytes]) -> Tuple[bytes, int]:
129
+ """Largest set of byte-identical payloads, ties to the earliest value.
130
+
131
+ ``values`` must already be ordered best source first.
132
+ """
133
+ counts = Counter(values)
134
+ best = max(counts.values())
135
+ tied = {value for value, n in counts.items() if n == best}
136
+ if len(tied) == 1:
137
+ return tied.pop(), best
138
+ return next(v for v in values if v in tied), best
139
+
140
+
141
+ def majority_bytes(values: Sequence[bytes]) -> bytes:
142
+ """Byte-wise majority over equal-length payloads.
143
+
144
+ Per byte position the most common value wins; a tie goes to the earliest
145
+ payload holding one of the tied values, so ``values`` must already be
146
+ ordered best source first.
147
+ """
148
+ out = bytearray()
149
+ for column in zip(*values, strict=True):
150
+ if len(set(column)) == 1:
151
+ out.append(column[0])
152
+ continue
153
+ counts = Counter(column)
154
+ best = max(counts.values())
155
+ winners = {value for value, n in counts.items() if n == best}
156
+ if len(winners) == 1:
157
+ out.append(winners.pop())
158
+ else:
159
+ out.append(next(v for v in column if v in winners))
160
+ return bytes(out)
161
+
162
+
163
+ def _check_trials(group: Sequence[Candidate]) -> List[Tuple[int, bytes]]:
164
+ """(mark, check) pairs a voted payload can be tested against.
165
+
166
+ The check bytes come off the same damaged track as the payload, so the
167
+ vote covers them too and the voted value is tried alongside the ones that
168
+ were actually read.
169
+ """
170
+ trials: List[Tuple[int, bytes]] = []
171
+ seen = set()
172
+ for cand in group:
173
+ if cand.check and (cand.mark, cand.check) not in seen:
174
+ seen.add((cand.mark, cand.check))
175
+ trials.append((cand.mark, cand.check))
176
+ with_check = [c for c in group if c.check]
177
+ if len(with_check) > 1:
178
+ size = Counter(len(c.check) for c in with_check).most_common(1)[0][0]
179
+ same = [c for c in with_check if len(c.check) == size]
180
+ voted = (Counter(c.mark for c in same).most_common(1)[0][0],
181
+ majority_bytes([c.check for c in same]))
182
+ if voted not in seen:
183
+ trials.append(voted)
184
+ return trials
185
+
186
+
187
+ def _per_source(group: Sequence[Candidate]) -> List[bytes]:
188
+ """One payload per input file, best source first."""
189
+ by_source: Dict[Path, List[bytes]] = {}
190
+ for cand in group:
191
+ by_source.setdefault(cand.source, []).append(cand.data)
192
+ return [majority_bytes(v) for v in by_source.values()]
193
+
194
+
195
+ def reconstructions(group: Sequence[Candidate]) -> Iterator[Tuple[str, bytes]]:
196
+ """Payloads worth testing against the check values that came off the disk.
197
+
198
+ The plain majority is what a vote means and comes first. The other two
199
+ cover what it cannot express: a capture with more revolutions than the
200
+ rest drowning out the sources that read the sector correctly, and a read
201
+ whose payload is fine but whose own check bytes were the damaged part.
202
+ """
203
+ payloads = [c.data for c in group]
204
+ plain, per_source = METHODS[:2]
205
+ seen = set()
206
+ for method, data in [(plain, majority_bytes(payloads)),
207
+ (per_source, majority_bytes(_per_source(group)))]:
208
+ if data not in seen:
209
+ seen.add(data)
210
+ yield method, data
211
+ for data in payloads:
212
+ if data not in seen:
213
+ seen.add(data)
214
+ yield METHODS[2], data
215
+
216
+
217
+ def unstable(group: Sequence[Candidate]) -> bool:
218
+ """True if some input read this sector two different ways on its own.
219
+
220
+ Two revolutions of one capture disagreeing is the signature of weak bits,
221
+ which are deliberate on protected disks and do not settle down however
222
+ many more times the disk is read. Comparing revolution against revolution
223
+ rather than pooling the whole file keeps a repeated ``--pll`` out of it:
224
+ two settings that read the same revolution two ways say something about
225
+ the decode, not about the disk.
226
+ """
227
+ per_source: Dict[Path, Dict[int, set]] = defaultdict(dict)
228
+ for cand in group:
229
+ per_source[cand.source].setdefault(cand.rev, set()).add(cand.data)
230
+ for by_rev in per_source.values():
231
+ revs = list(by_rev.values())
232
+ if len(revs) > 1 and set.union(*revs) != set.intersection(*revs):
233
+ return True
234
+ return False
235
+
236
+
237
+ def resolve(key: Tuple[int, int, int], size: int,
238
+ group: Sequence[Candidate], vote: bool = True) -> Resolution:
239
+ """Apply the three tiers to one sector's read attempts."""
240
+ cyl, head, sec_id = key
241
+ usable = [c for c in group if len(c.data) == size]
242
+ res = Resolution(cyl=cyl, head=head, sec_id=sec_id, size=size,
243
+ data=make_filler(size), status=MISSING,
244
+ attempts=len(group), discarded=len(group) - len(usable))
245
+ if not usable:
246
+ return res
247
+ res.good = sum(1 for c in usable if c.data_crc_ok)
248
+ res.unstable = unstable(usable)
249
+
250
+ clean = [c for c in usable if c.data_crc_ok]
251
+ if clean:
252
+ # A read that satisfied a check value off the disk outranks one that
253
+ # was merely never contradicted, which is all a sector image can say.
254
+ verified = [c for c in clean if c.check]
255
+ data, _ = largest_cluster([c.data for c in verified or clean])
256
+ matched = [c for c in clean if c.data == data]
257
+ res.data, res.status = data, CLEAN
258
+ res.agreement = len(matched)
259
+ res.verified = any(c.check for c in matched)
260
+ res.contested = len(matched) < len(clean)
261
+ res.sources = [Contribution(c.source, c.rev) for c in matched]
262
+ return res
263
+
264
+ if vote and len(usable) > 1:
265
+ codec = usable[0].codec
266
+ trials = _check_trials(usable)
267
+ for method, data in reconstructions(usable):
268
+ if any(verify_sector(codec, mark, data, check)
269
+ for mark, check in trials):
270
+ res.data, res.status, res.method = data, VOTED, method
271
+ res.verified = True
272
+ matched = [c for c in usable if c.data == data]
273
+ res.agreement = len(matched)
274
+ res.sources = [Contribution(c.source, c.rev)
275
+ for c in matched or usable]
276
+ return res
277
+
278
+ data, agree = largest_cluster([c.data for c in usable])
279
+ res.data, res.status, res.agreement = data, UNRESOLVED, agree
280
+ res.sources = [Contribution(c.source, c.rev) for c in usable
281
+ if c.data == data]
282
+ return res
283
+
284
+
285
+ def stack(candidates: Iterable[Candidate],
286
+ expected: Sequence[Tuple[int, int, int]],
287
+ sizes: Dict[Tuple[int, int, int], int],
288
+ order: Sequence[Path],
289
+ vote: bool = True) -> StackResult:
290
+ """Merge every read attempt into one resolution per expected sector."""
291
+ candidates = list(candidates)
292
+ ranking = rank_sources(candidates, order)
293
+ rank = {path: i for i, path in enumerate(ranking)}
294
+
295
+ groups: Dict[Tuple[int, int, int], List[Candidate]] = defaultdict(list)
296
+ for cand in candidates:
297
+ groups[cand.key].append(cand)
298
+ for group in groups.values():
299
+ group.sort(key=lambda c: (rank.get(c.source, len(rank)), c.rev))
300
+
301
+ resolutions = [resolve(key, sizes[key], groups.get(key, ()), vote=vote)
302
+ for key in expected]
303
+
304
+ supplied = {path: 0 for path in order}
305
+ sole = {path: 0 for path in order}
306
+ for res in resolutions:
307
+ if not res.resolved:
308
+ continue
309
+ paths = {c.source for c in res.sources}
310
+ for path in paths:
311
+ supplied[path] = supplied.get(path, 0) + 1
312
+ if len(paths) == 1:
313
+ path = next(iter(paths))
314
+ sole[path] = sole.get(path, 0) + 1
315
+
316
+ return StackResult(resolutions=resolutions, ranking=ranking,
317
+ supplied=supplied, sole_source=sole)