bindery-cli 0.27.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bindery/__init__.py +12 -0
- bindery/__main__.py +4 -0
- bindery/audit.py +2137 -0
- bindery/cli.py +961 -0
- bindery/epub.py +618 -0
- bindery/library.py +244 -0
- bindery/pagination.py +359 -0
- bindery/reserialize.py +42 -0
- bindery/transforms.py +579 -0
- bindery/validate.py +254 -0
- bindery/watermark.py +244 -0
- bindery_cli-0.27.0.dist-info/METADATA +218 -0
- bindery_cli-0.27.0.dist-info/RECORD +16 -0
- bindery_cli-0.27.0.dist-info/WHEEL +4 -0
- bindery_cli-0.27.0.dist-info/entry_points.txt +2 -0
- bindery_cli-0.27.0.dist-info/licenses/LICENSE +21 -0
bindery/audit.py
ADDED
|
@@ -0,0 +1,2137 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
bindery audit: read the actual text of every EPUB and flag content problems that
|
|
4
|
+
metadata and structural validators cannot see. Four analyzers, one tool:
|
|
5
|
+
|
|
6
|
+
content non-English bodies (wrong-language editions) and injected
|
|
7
|
+
foreign-language ad-notices (declared lang=eng, body Portuguese
|
|
8
|
+
/ Russian / etc.)
|
|
9
|
+
pagenumbers print page numbers (and running headers) captured as body *text*
|
|
10
|
+
by a bad PDF/OCR conversion, so they reflow into the middle of a
|
|
11
|
+
sentence ("where the hay cart 16 was taking him")
|
|
12
|
+
emptytext content-less stubs: a "Bookmate" export is cover/promo images
|
|
13
|
+
plus a tiny HTML placeholder, the spine pointing only at the
|
|
14
|
+
placeholder, the book itself absent (passes epubcheck and a
|
|
15
|
+
structural repairer because the one referenced doc is valid)
|
|
16
|
+
ocr OCR/conversion-damaged prose: paragraphs split mid-sentence
|
|
17
|
+
("could just make out the shape" / "of another boat"), plus
|
|
18
|
+
dictionary-free side signals (en-dashes inside words, doubled
|
|
19
|
+
opening quotes, space-stripped proper nouns recurring alongside
|
|
20
|
+
their hyphenated form). Character-substitution errors ("sonic"
|
|
21
|
+
for "some") are OUT OF SCOPE: catching those needs a wordlist,
|
|
22
|
+
and this tool is stdlib-only by contract.
|
|
23
|
+
all run all four in a SINGLE decompression pass per book
|
|
24
|
+
|
|
25
|
+
This merges the former audit_epub_content.py / audit_epub_pagenumbers.py /
|
|
26
|
+
audit_epub_emptytext.py: they shared the same spine resolution, library/
|
|
27
|
+
directory dual-mode, read-only contract, and exit codes, and differed only in
|
|
28
|
+
the per-book verdict. `all` opens each EPUB once and feeds the decoded spine to
|
|
29
|
+
all four analyzers (the expensive part is decompression, so this is a real
|
|
30
|
+
win at library scale).
|
|
31
|
+
|
|
32
|
+
Companion to validate_metadata.py (which audits the catalogue) and to Bindery
|
|
33
|
+
(which repairs EPUB structure). This one reads body text and changes nothing;
|
|
34
|
+
it opens metadata.db strictly mode=ro.
|
|
35
|
+
|
|
36
|
+
Run from the library directory:
|
|
37
|
+
python3 bindery audit all # all four audits, whole library
|
|
38
|
+
python3 bindery audit content # one audit, whole library
|
|
39
|
+
python3 bindery audit all ~/Downloads # vet loose .epub files before import
|
|
40
|
+
python3 bindery audit emptytext ~/Downloads --min-chars 1000
|
|
41
|
+
|
|
42
|
+
Library mode pulls the EPUB list (and tags / declared language) from
|
|
43
|
+
metadata.db; directory mode scans every .epub it finds recursively, the
|
|
44
|
+
workflow for checking downloads before they enter the library.
|
|
45
|
+
|
|
46
|
+
Exit codes:
|
|
47
|
+
0 = clean (THIN empty-text hits are advisory and do not fail the run)
|
|
48
|
+
1 = a real problem found (foreign content, baked page numbers, empty book,
|
|
49
|
+
OCR-damaged prose) or a scan error
|
|
50
|
+
2 = setup error (missing DB / library, or no .epub files in directory)
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
import argparse
|
|
54
|
+
import sys
|
|
55
|
+
from pathlib import Path
|
|
56
|
+
|
|
57
|
+
sys.path.insert(0, str(Path(__file__).parent))
|
|
58
|
+
import os
|
|
59
|
+
import re
|
|
60
|
+
import sys
|
|
61
|
+
import zipfile
|
|
62
|
+
from collections import Counter
|
|
63
|
+
from pathlib import Path
|
|
64
|
+
from xml.etree import ElementTree as ET
|
|
65
|
+
|
|
66
|
+
import vir_tui as ui
|
|
67
|
+
|
|
68
|
+
# ----------------------------------------------------------------------------
|
|
69
|
+
# Shared scaffolding
|
|
70
|
+
# ----------------------------------------------------------------------------
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def resolve_library_root() -> Path | None:
|
|
74
|
+
"""The library root is wherever metadata.db sits: next to this script (the
|
|
75
|
+
copy living inside the library) or the current working directory (running
|
|
76
|
+
the repo copy from inside a library), in that order."""
|
|
77
|
+
for d in (Path(__file__).resolve().parent, Path.cwd()):
|
|
78
|
+
if (d / "metadata.db").is_file():
|
|
79
|
+
return d
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ANSI colours; suppress when stdout isn't a TTY.
|
|
84
|
+
USE_COLOR = sys.stdout.isatty()
|
|
85
|
+
RED = "\033[31m" if USE_COLOR else ""
|
|
86
|
+
YELLOW = "\033[33m" if USE_COLOR else ""
|
|
87
|
+
GREEN = "\033[32m" if USE_COLOR else ""
|
|
88
|
+
BOLD = "\033[1m" if USE_COLOR else ""
|
|
89
|
+
RESET = "\033[0m" if USE_COLOR else ""
|
|
90
|
+
|
|
91
|
+
CONTAINER_NS = {"c": "urn:oasis:names:tc:opendocument:xmlns:container"}
|
|
92
|
+
OPF_NS = "{http://www.idpf.org/2007/opf}"
|
|
93
|
+
|
|
94
|
+
_PCT = re.compile(r"(?:%[0-9A-Fa-f]{2})+")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _pct_decode(href: str) -> str:
|
|
98
|
+
"""Percent-decode an OPF href (UTF-8). OPF manifest hrefs are IRIs, so a
|
|
99
|
+
reserved char like '!' is written '%21' and a multi-byte char as a run of
|
|
100
|
+
%XX (e.g. 'ö' -> '%C3%B6'); each run must be decoded as bytes together.
|
|
101
|
+
Stdlib-only stand-in for urllib.parse.unquote; invalid escapes stay literal."""
|
|
102
|
+
return _PCT.sub(
|
|
103
|
+
lambda m: bytes.fromhex(m.group(0).replace("%", "")).decode("utf-8", "replace"),
|
|
104
|
+
href,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class Book:
|
|
109
|
+
"""A decompressed EPUB: spine documents read once and shared by every
|
|
110
|
+
analyzer. The whole point of the single-pass design lives here."""
|
|
111
|
+
|
|
112
|
+
__slots__ = (
|
|
113
|
+
"_visible",
|
|
114
|
+
"corrupt",
|
|
115
|
+
"docs",
|
|
116
|
+
"lang",
|
|
117
|
+
"names",
|
|
118
|
+
"nav",
|
|
119
|
+
"spine",
|
|
120
|
+
"toc_refs",
|
|
121
|
+
"toc_absent",
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
def __init__(self, spine, nav, lang, docs, names, corrupt, toc_refs, toc_absent):
|
|
125
|
+
self.spine = spine # resolved, in-order, in-archive spine doc paths
|
|
126
|
+
self.nav = nav # the nav document path, or None
|
|
127
|
+
self.lang = lang # declared dc:language (lowercased), or ""
|
|
128
|
+
self.docs = docs # {path: decoded html} for every spine doc
|
|
129
|
+
self.names = names # full archive namelist (for image / marker counts)
|
|
130
|
+
self.corrupt = corrupt # entries whose full read failed (CRC/truncation)
|
|
131
|
+
# Spine-integrity accounting (phase 8): nav/NCX ToC references vs what
|
|
132
|
+
# the archive actually contains. The official Wandering Inn builds
|
|
133
|
+
# ship series-wide ToC manifests (~750 references vs 14-19 real
|
|
134
|
+
# content docs) — reported as `convention`, not flagged.
|
|
135
|
+
self.toc_refs = toc_refs
|
|
136
|
+
self.toc_absent = toc_absent
|
|
137
|
+
self._visible: list[str] | None = None
|
|
138
|
+
|
|
139
|
+
def visible_texts(self) -> list[str]:
|
|
140
|
+
"""Rendered text, one entry per spine doc, computed once per book.
|
|
141
|
+
|
|
142
|
+
emptytext and ocr both need it, and under `all` they each used to strip
|
|
143
|
+
tags over the whole book independently: the second pass was pure waste
|
|
144
|
+
in a design whose entire point is touching each EPUB once.
|
|
145
|
+
"""
|
|
146
|
+
if self._visible is None:
|
|
147
|
+
self._visible = [_visible_text(self.docs.get(d, "")) for d in self.spine]
|
|
148
|
+
return self._visible
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def load_book(path: Path) -> Book:
|
|
152
|
+
"""Open an EPUB once: fully read EVERY archive entry (the CRC sweep the
|
|
153
|
+
phase-1 skill used to do by hand), resolve spine + nav + declared
|
|
154
|
+
language, and read every spine document's HTML. Decoded utf-8 with
|
|
155
|
+
replacement (never raises); a corrupted archive entry (bad CRC, truncated
|
|
156
|
+
stream) is recorded in `Book.corrupt` and reads as empty text — the
|
|
157
|
+
corruption verdict, not emptytext, owns that story."""
|
|
158
|
+
with zipfile.ZipFile(path) as z:
|
|
159
|
+
names = z.namelist()
|
|
160
|
+
nameset = set(names)
|
|
161
|
+
corrupt: list[str] = []
|
|
162
|
+
raw: dict[str, bytes] = {}
|
|
163
|
+
|
|
164
|
+
def _read(name: str) -> bytes | None:
|
|
165
|
+
"""Fully read one entry (CRC + real decompression, not just the
|
|
166
|
+
central directory's word). A corrupt entry is recorded and its
|
|
167
|
+
content treated as absent — never silently swallowed."""
|
|
168
|
+
if name in raw:
|
|
169
|
+
return raw[name]
|
|
170
|
+
if name in corrupt:
|
|
171
|
+
return None
|
|
172
|
+
try:
|
|
173
|
+
blob = z.read(name)
|
|
174
|
+
except Exception:
|
|
175
|
+
corrupt.append(name)
|
|
176
|
+
return None
|
|
177
|
+
raw[name] = blob
|
|
178
|
+
return blob
|
|
179
|
+
|
|
180
|
+
container = ET.fromstring(_read("META-INF/container.xml"))
|
|
181
|
+
rootfile = container.find(".//c:rootfile", CONTAINER_NS)
|
|
182
|
+
opf_path = rootfile.get("full-path") if rootfile is not None else None
|
|
183
|
+
if not opf_path:
|
|
184
|
+
raise ValueError("container.xml has no rootfile")
|
|
185
|
+
opf = ET.fromstring(_read(opf_path))
|
|
186
|
+
base = os.path.dirname(opf_path)
|
|
187
|
+
|
|
188
|
+
manifest: dict[str, str] = {}
|
|
189
|
+
nav_href: str | None = None
|
|
190
|
+
for it in opf.iter(OPF_NS + "item"):
|
|
191
|
+
item_id, href = it.get("id"), it.get("href")
|
|
192
|
+
if not item_id or not href:
|
|
193
|
+
continue
|
|
194
|
+
manifest[item_id] = href
|
|
195
|
+
if "nav" in (it.get("properties") or ""):
|
|
196
|
+
nav_href = href
|
|
197
|
+
|
|
198
|
+
lang = ""
|
|
199
|
+
for el in opf.iter():
|
|
200
|
+
if el.tag.endswith("}language") and el.text:
|
|
201
|
+
lang = el.text.strip().lower()
|
|
202
|
+
break
|
|
203
|
+
|
|
204
|
+
def full(href: str) -> str:
|
|
205
|
+
# Decode the percent-encoded IRI and drop any #fragment before
|
|
206
|
+
# matching the archive namelist; otherwise a spine doc whose
|
|
207
|
+
# filename has a reserved char (e.g. '!' written '%21') resolves to
|
|
208
|
+
# nothing and the book reads as empty (false EMPTY verdict).
|
|
209
|
+
href = _pct_decode(href.split("#", 1)[0])
|
|
210
|
+
return os.path.normpath(f"{base}/{href}" if base else href).replace(
|
|
211
|
+
"\\", "/"
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
spine: list[str] = []
|
|
215
|
+
for itemref in opf.iter(OPF_NS + "itemref"):
|
|
216
|
+
idref = itemref.get("idref")
|
|
217
|
+
href = manifest.get(idref) if idref else None
|
|
218
|
+
if href and full(href) in nameset:
|
|
219
|
+
spine.append(full(href))
|
|
220
|
+
nav = full(nav_href) if nav_href else None
|
|
221
|
+
|
|
222
|
+
# The single full pass: every archive entry is fully read here (the
|
|
223
|
+
# CRC sweep), so the text analysis below runs on what actually
|
|
224
|
+
# decompressed. Corrupt entries are named, not averaged away.
|
|
225
|
+
for name in names:
|
|
226
|
+
if name.endswith("/"):
|
|
227
|
+
continue
|
|
228
|
+
_read(name)
|
|
229
|
+
|
|
230
|
+
docs: dict[str, str] = {}
|
|
231
|
+
for doc in spine:
|
|
232
|
+
blob = raw.get(doc)
|
|
233
|
+
docs[doc] = blob.decode("utf-8", "replace") if blob is not None else ""
|
|
234
|
+
# ToC reference accounting: every nav/NCX anchor target checked against
|
|
235
|
+
# the archive (manifest items are the spine's own source and already
|
|
236
|
+
# accounted by the spine resolution).
|
|
237
|
+
toc_refs = 0
|
|
238
|
+
toc_absent = 0
|
|
239
|
+
|
|
240
|
+
def _account(href: str) -> None:
|
|
241
|
+
nonlocal toc_refs, toc_absent
|
|
242
|
+
href = href.strip()
|
|
243
|
+
if href.startswith("#"):
|
|
244
|
+
return # in-page anchor, not an archive target
|
|
245
|
+
if re.match(r"^[a-z]+:", href, re.IGNORECASE):
|
|
246
|
+
return # foreign URL scheme, not an archive target
|
|
247
|
+
toc_refs += 1
|
|
248
|
+
if full(href) not in nameset:
|
|
249
|
+
toc_absent += 1
|
|
250
|
+
|
|
251
|
+
nav_html = docs.get(nav) if nav else None
|
|
252
|
+
if nav_html is None and nav:
|
|
253
|
+
# The nav document is usually in the spine, but nothing requires it.
|
|
254
|
+
blob = _read(nav)
|
|
255
|
+
nav_html = blob.decode("utf-8", "replace") if blob is not None else ""
|
|
256
|
+
if nav_html:
|
|
257
|
+
for m in re.finditer(r'href\s*=\s*["\']([^"\']+)', nav_html):
|
|
258
|
+
_account(m.group(1))
|
|
259
|
+
ncx_path = None
|
|
260
|
+
for it in opf.iter(OPF_NS + "item"):
|
|
261
|
+
if (it.get("media-type") or "").lower() == "application/x-dtbncx+xml":
|
|
262
|
+
ncx_path = full(it.get("href") or "")
|
|
263
|
+
break
|
|
264
|
+
if ncx_path:
|
|
265
|
+
blob = _read(ncx_path)
|
|
266
|
+
if blob is not None:
|
|
267
|
+
for m in re.finditer(
|
|
268
|
+
r'<content[^>]+src\s*=\s*["\']([^"\']+)',
|
|
269
|
+
blob.decode("utf-8", "replace"),
|
|
270
|
+
):
|
|
271
|
+
_account(m.group(1))
|
|
272
|
+
|
|
273
|
+
return Book(spine, nav, lang, docs, names, corrupt, toc_refs, toc_absent)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
# ----------------------------------------------------------------------------
|
|
277
|
+
# Analyzer: content (non-English / injected notices)
|
|
278
|
+
# ----------------------------------------------------------------------------
|
|
279
|
+
|
|
280
|
+
CAP = 400_000 # clean chars read per book; ample for a language verdict
|
|
281
|
+
|
|
282
|
+
STYLE_RE = re.compile(r"<(style|script)\b[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
|
|
283
|
+
TAG_RE = re.compile(r"<[^>]+>")
|
|
284
|
+
WS_RE = re.compile(r"\s+")
|
|
285
|
+
WORD_RE = re.compile(r"[a-zA-ZàâäéèêëïîôöùûüçñáíóúãõßÀ-ÿ']+")
|
|
286
|
+
SIGNATURE_RE = re.compile(r"importknig|книжный импорт|knizhny", re.IGNORECASE)
|
|
287
|
+
|
|
288
|
+
# Distinctive stopword sets. Book-length text makes the vote unambiguous;
|
|
289
|
+
# the small EN/IT overlap on "i" etc. is swamped by the rest.
|
|
290
|
+
STOPWORDS: dict[str, set[str]] = {
|
|
291
|
+
"en": set(
|
|
292
|
+
[
|
|
293
|
+
"the",
|
|
294
|
+
"of",
|
|
295
|
+
"and",
|
|
296
|
+
"to",
|
|
297
|
+
"a",
|
|
298
|
+
"in",
|
|
299
|
+
"that",
|
|
300
|
+
"is",
|
|
301
|
+
"was",
|
|
302
|
+
"for",
|
|
303
|
+
"it",
|
|
304
|
+
"with",
|
|
305
|
+
"as",
|
|
306
|
+
"his",
|
|
307
|
+
"on",
|
|
308
|
+
"be",
|
|
309
|
+
"at",
|
|
310
|
+
"by",
|
|
311
|
+
"he",
|
|
312
|
+
"this",
|
|
313
|
+
"had",
|
|
314
|
+
"not",
|
|
315
|
+
"are",
|
|
316
|
+
"but",
|
|
317
|
+
"from",
|
|
318
|
+
"or",
|
|
319
|
+
"have",
|
|
320
|
+
"an",
|
|
321
|
+
"they",
|
|
322
|
+
"which",
|
|
323
|
+
"one",
|
|
324
|
+
"you",
|
|
325
|
+
"were",
|
|
326
|
+
"her",
|
|
327
|
+
"all",
|
|
328
|
+
"she",
|
|
329
|
+
"there",
|
|
330
|
+
"would",
|
|
331
|
+
"their",
|
|
332
|
+
]
|
|
333
|
+
),
|
|
334
|
+
"de": set(
|
|
335
|
+
[
|
|
336
|
+
"der",
|
|
337
|
+
"die",
|
|
338
|
+
"und",
|
|
339
|
+
"in",
|
|
340
|
+
"den",
|
|
341
|
+
"von",
|
|
342
|
+
"zu",
|
|
343
|
+
"das",
|
|
344
|
+
"mit",
|
|
345
|
+
"sich",
|
|
346
|
+
"des",
|
|
347
|
+
"auf",
|
|
348
|
+
"für",
|
|
349
|
+
"ist",
|
|
350
|
+
"im",
|
|
351
|
+
"dem",
|
|
352
|
+
"nicht",
|
|
353
|
+
"ein",
|
|
354
|
+
"eine",
|
|
355
|
+
"als",
|
|
356
|
+
"auch",
|
|
357
|
+
"es",
|
|
358
|
+
"an",
|
|
359
|
+
"werden",
|
|
360
|
+
"aus",
|
|
361
|
+
"er",
|
|
362
|
+
"hat",
|
|
363
|
+
"dass",
|
|
364
|
+
"sie",
|
|
365
|
+
"nach",
|
|
366
|
+
"wird",
|
|
367
|
+
"bei",
|
|
368
|
+
"einer",
|
|
369
|
+
"um",
|
|
370
|
+
]
|
|
371
|
+
),
|
|
372
|
+
"fr": set(
|
|
373
|
+
[
|
|
374
|
+
"le",
|
|
375
|
+
"la",
|
|
376
|
+
"les",
|
|
377
|
+
"de",
|
|
378
|
+
"des",
|
|
379
|
+
"un",
|
|
380
|
+
"une",
|
|
381
|
+
"et",
|
|
382
|
+
"en",
|
|
383
|
+
"dans",
|
|
384
|
+
"que",
|
|
385
|
+
"qui",
|
|
386
|
+
"pour",
|
|
387
|
+
"pas",
|
|
388
|
+
"sur",
|
|
389
|
+
"au",
|
|
390
|
+
"avec",
|
|
391
|
+
"ce",
|
|
392
|
+
"il",
|
|
393
|
+
"ne",
|
|
394
|
+
"se",
|
|
395
|
+
"plus",
|
|
396
|
+
"par",
|
|
397
|
+
"je",
|
|
398
|
+
"nous",
|
|
399
|
+
"vous",
|
|
400
|
+
"est",
|
|
401
|
+
"son",
|
|
402
|
+
"ses",
|
|
403
|
+
"aux",
|
|
404
|
+
]
|
|
405
|
+
),
|
|
406
|
+
"es": set(
|
|
407
|
+
[
|
|
408
|
+
"el",
|
|
409
|
+
"la",
|
|
410
|
+
"los",
|
|
411
|
+
"las",
|
|
412
|
+
"de",
|
|
413
|
+
"un",
|
|
414
|
+
"una",
|
|
415
|
+
"y",
|
|
416
|
+
"en",
|
|
417
|
+
"que",
|
|
418
|
+
"no",
|
|
419
|
+
"se",
|
|
420
|
+
"con",
|
|
421
|
+
"por",
|
|
422
|
+
"para",
|
|
423
|
+
"es",
|
|
424
|
+
"su",
|
|
425
|
+
"lo",
|
|
426
|
+
"como",
|
|
427
|
+
"más",
|
|
428
|
+
"pero",
|
|
429
|
+
"sus",
|
|
430
|
+
"le",
|
|
431
|
+
"ya",
|
|
432
|
+
"o",
|
|
433
|
+
"este",
|
|
434
|
+
"sí",
|
|
435
|
+
"porque",
|
|
436
|
+
"esta",
|
|
437
|
+
"entre",
|
|
438
|
+
]
|
|
439
|
+
),
|
|
440
|
+
"it": set(
|
|
441
|
+
[
|
|
442
|
+
"il",
|
|
443
|
+
"lo",
|
|
444
|
+
"la",
|
|
445
|
+
"i",
|
|
446
|
+
"gli",
|
|
447
|
+
"le",
|
|
448
|
+
"di",
|
|
449
|
+
"un",
|
|
450
|
+
"uno",
|
|
451
|
+
"una",
|
|
452
|
+
"e",
|
|
453
|
+
"che",
|
|
454
|
+
"non",
|
|
455
|
+
"per",
|
|
456
|
+
"con",
|
|
457
|
+
"su",
|
|
458
|
+
"come",
|
|
459
|
+
"più",
|
|
460
|
+
"ma",
|
|
461
|
+
"anche",
|
|
462
|
+
"da",
|
|
463
|
+
"sono",
|
|
464
|
+
"mi",
|
|
465
|
+
"si",
|
|
466
|
+
"nel",
|
|
467
|
+
"alla",
|
|
468
|
+
"dei",
|
|
469
|
+
"delle",
|
|
470
|
+
]
|
|
471
|
+
),
|
|
472
|
+
"pt": set(
|
|
473
|
+
[
|
|
474
|
+
"o",
|
|
475
|
+
"a",
|
|
476
|
+
"os",
|
|
477
|
+
"as",
|
|
478
|
+
"de",
|
|
479
|
+
"um",
|
|
480
|
+
"uma",
|
|
481
|
+
"e",
|
|
482
|
+
"que",
|
|
483
|
+
"do",
|
|
484
|
+
"da",
|
|
485
|
+
"em",
|
|
486
|
+
"não",
|
|
487
|
+
"se",
|
|
488
|
+
"com",
|
|
489
|
+
"por",
|
|
490
|
+
"para",
|
|
491
|
+
"mais",
|
|
492
|
+
"mas",
|
|
493
|
+
"como",
|
|
494
|
+
"ao",
|
|
495
|
+
"dos",
|
|
496
|
+
"das",
|
|
497
|
+
"na",
|
|
498
|
+
"no",
|
|
499
|
+
"à",
|
|
500
|
+
"seu",
|
|
501
|
+
]
|
|
502
|
+
),
|
|
503
|
+
"nl": set(
|
|
504
|
+
[
|
|
505
|
+
"de",
|
|
506
|
+
"het",
|
|
507
|
+
"een",
|
|
508
|
+
"en",
|
|
509
|
+
"van",
|
|
510
|
+
"te",
|
|
511
|
+
"dat",
|
|
512
|
+
"die",
|
|
513
|
+
"in",
|
|
514
|
+
"is",
|
|
515
|
+
"op",
|
|
516
|
+
"ik",
|
|
517
|
+
"niet",
|
|
518
|
+
"met",
|
|
519
|
+
"zijn",
|
|
520
|
+
"er",
|
|
521
|
+
"maar",
|
|
522
|
+
"om",
|
|
523
|
+
"ook",
|
|
524
|
+
"als",
|
|
525
|
+
"voor",
|
|
526
|
+
"naar",
|
|
527
|
+
"dan",
|
|
528
|
+
"zou",
|
|
529
|
+
"hij",
|
|
530
|
+
"heeft",
|
|
531
|
+
]
|
|
532
|
+
),
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def script_of(codepoint: int) -> str | None:
|
|
537
|
+
if 0x0400 <= codepoint <= 0x04FF:
|
|
538
|
+
return "Cyrillic"
|
|
539
|
+
if 0x4E00 <= codepoint <= 0x9FFF:
|
|
540
|
+
return "CJK-Han"
|
|
541
|
+
if 0x3040 <= codepoint <= 0x30FF:
|
|
542
|
+
return "Japanese-kana"
|
|
543
|
+
if 0xAC00 <= codepoint <= 0xD7A3:
|
|
544
|
+
return "Korean"
|
|
545
|
+
if 0x0600 <= codepoint <= 0x06FF:
|
|
546
|
+
return "Arabic"
|
|
547
|
+
if 0x0370 <= codepoint <= 0x03FF:
|
|
548
|
+
return "Greek"
|
|
549
|
+
if 0x0590 <= codepoint <= 0x05FF:
|
|
550
|
+
return "Hebrew"
|
|
551
|
+
if 0x0900 <= codepoint <= 0x097F:
|
|
552
|
+
return "Devanagari"
|
|
553
|
+
return None
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _content_clean(html: str) -> str:
|
|
557
|
+
html = STYLE_RE.sub(" ", html)
|
|
558
|
+
return WS_RE.sub(" ", TAG_RE.sub(" ", html))
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def analyze_content(book: Book) -> dict:
|
|
562
|
+
"""Language / script signal over the (pre-read) spine."""
|
|
563
|
+
docs = book.spine
|
|
564
|
+
parts: list[str] = []
|
|
565
|
+
total = 0
|
|
566
|
+
for doc in docs:
|
|
567
|
+
if total >= CAP:
|
|
568
|
+
break
|
|
569
|
+
text = _content_clean(book.docs.get(doc, ""))
|
|
570
|
+
parts.append(text)
|
|
571
|
+
total += len(text)
|
|
572
|
+
if docs and docs[-1] not in docs[: len(parts)]:
|
|
573
|
+
parts.append(_content_clean(book.docs.get(docs[-1], "")))
|
|
574
|
+
|
|
575
|
+
text = " ".join(parts)
|
|
576
|
+
# Count scripts over the first 250k letters in a single pass, instead of
|
|
577
|
+
# materializing a list of every letter's codepoint in a book-length string.
|
|
578
|
+
scripts: Counter = Counter()
|
|
579
|
+
total_letters = 0
|
|
580
|
+
for c in text:
|
|
581
|
+
if c.isalpha():
|
|
582
|
+
total_letters += 1
|
|
583
|
+
s = script_of(ord(c))
|
|
584
|
+
if s:
|
|
585
|
+
scripts[s] += 1
|
|
586
|
+
if total_letters >= 250_000:
|
|
587
|
+
break
|
|
588
|
+
nonlatin = sum(scripts.values())
|
|
589
|
+
total_letters = total_letters or 1
|
|
590
|
+
words = WORD_RE.findall(text.lower())[:5000]
|
|
591
|
+
ratios = {
|
|
592
|
+
code: (sum(w in stops for w in words) / len(words) if words else 0.0)
|
|
593
|
+
for code, stops in STOPWORDS.items()
|
|
594
|
+
}
|
|
595
|
+
best = max(ratios, key=lambda c: ratios[c])
|
|
596
|
+
return {
|
|
597
|
+
"lang": book.lang,
|
|
598
|
+
"scripts": dict(scripts),
|
|
599
|
+
"nonlatin": nonlatin,
|
|
600
|
+
"nonlatin_frac": nonlatin / total_letters,
|
|
601
|
+
"ratios": ratios,
|
|
602
|
+
"best": best,
|
|
603
|
+
"nwords": len(words),
|
|
604
|
+
"signature": bool(SIGNATURE_RE.search(text)),
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def findings(r: dict) -> list[tuple[str, str]]:
|
|
609
|
+
"""Classify a content result into [(category, detail)]; empty = English and clean."""
|
|
610
|
+
out: list[tuple[str, str]] = []
|
|
611
|
+
if r["nonlatin"] >= 150 and r["nonlatin_frac"] > 0.02:
|
|
612
|
+
top = max(r["scripts"], key=lambda s: r["scripts"][s])
|
|
613
|
+
out.append(
|
|
614
|
+
(
|
|
615
|
+
"NON-LATIN SCRIPT",
|
|
616
|
+
f"{r['nonlatin_frac'] * 100:.0f}% {top} ({r['nonlatin']} non-Latin letters)",
|
|
617
|
+
)
|
|
618
|
+
)
|
|
619
|
+
if (
|
|
620
|
+
r["best"] != "en"
|
|
621
|
+
and r["nwords"] >= 400
|
|
622
|
+
and r["ratios"][r["best"]] > 0.06
|
|
623
|
+
and r["ratios"][r["best"]] > 1.3 * r["ratios"]["en"]
|
|
624
|
+
):
|
|
625
|
+
top3 = ", ".join(
|
|
626
|
+
f"{c}={v:.2f}"
|
|
627
|
+
for c, v in sorted(r["ratios"].items(), key=lambda x: -x[1])[:3]
|
|
628
|
+
)
|
|
629
|
+
out.append(("LATIN-SCRIPT FOREIGN", f"looks {r['best'].upper()} [{top3}]"))
|
|
630
|
+
if r["signature"]:
|
|
631
|
+
out.append(
|
|
632
|
+
("INJECTION SIGNATURE", "importknig / Книжный импорт signature present")
|
|
633
|
+
)
|
|
634
|
+
return out
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def scan_content(path: Path) -> dict:
|
|
638
|
+
"""Convenience: load + analyze a single file (used by tests / ad-hoc runs)."""
|
|
639
|
+
return analyze_content(load_book(path))
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
# ----------------------------------------------------------------------------
|
|
643
|
+
# Analyzer: pagenumbers (print page numbers baked into body text)
|
|
644
|
+
# ----------------------------------------------------------------------------
|
|
645
|
+
|
|
646
|
+
INT_RE = re.compile(r"\d{1,4}$")
|
|
647
|
+
ROMAN_RE = re.compile(r"[ivxlcdm]{2,7}$", re.IGNORECASE)
|
|
648
|
+
# Block-level elements we track to reconstruct reading order.
|
|
649
|
+
BLOCK_TAGS = {
|
|
650
|
+
"p",
|
|
651
|
+
"div",
|
|
652
|
+
"h1",
|
|
653
|
+
"h2",
|
|
654
|
+
"h3",
|
|
655
|
+
"h4",
|
|
656
|
+
"h5",
|
|
657
|
+
"h6",
|
|
658
|
+
"li",
|
|
659
|
+
"td",
|
|
660
|
+
"th",
|
|
661
|
+
"blockquote",
|
|
662
|
+
"section",
|
|
663
|
+
"caption",
|
|
664
|
+
"figcaption",
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
# Tuning, validated by hand against a 3,897-EPUB library (every flagged book and
|
|
668
|
+
# the full n>=2 tail inspected). At these values the detector flagged 21 books,
|
|
669
|
+
# all true positives; the false-positive tail (experimental footnote-poems,
|
|
670
|
+
# scraped web-serial vote counts, placeholder section labels) all fell under
|
|
671
|
+
# MIN_BAKED_HITS or MIN_SPAN. Lowering MIN_BAKED_HITS to catch the few remaining
|
|
672
|
+
# 3-4 hit books (a localized stray-number patch) starts admitting those.
|
|
673
|
+
PROSE_MIN = 120 # a neighbour this long counts as a prose paragraph
|
|
674
|
+
RUNHEAD_MIN_REPEAT = 8 # a short block repeated this often is a running header
|
|
675
|
+
RUNHEAD_MAX_LEN = 60 # running headers are short
|
|
676
|
+
MIN_BAKED_HITS = 5 # below this, a handful of hits is too often coincidence
|
|
677
|
+
MIN_SPAN = 0.10 # flagged numbers must cover this fraction of the book (drops
|
|
678
|
+
# localized clusters: footnote-poems, scraped comment sections)
|
|
679
|
+
MIN_RUN = 1 # ascending run is informative but not gated; the baked test already
|
|
680
|
+
# requires genuine sentence interruption, so a short run is not disqualifying
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def roman_value(s: str) -> int | None:
|
|
684
|
+
vals = {"i": 1, "v": 5, "x": 10, "l": 50, "c": 100, "d": 500, "m": 1000}
|
|
685
|
+
total = 0
|
|
686
|
+
s = s.lower()
|
|
687
|
+
for i, c in enumerate(s):
|
|
688
|
+
if c not in vals:
|
|
689
|
+
return None
|
|
690
|
+
v = vals[c]
|
|
691
|
+
total += -v if (i + 1 < len(s) and vals[s[i + 1]] > v) else v
|
|
692
|
+
return total or None
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def number_value(text: str) -> int | None:
|
|
696
|
+
"""A bare page-number-ish value (1-9999 arabic, or a roman numeral), else None."""
|
|
697
|
+
if INT_RE.fullmatch(text):
|
|
698
|
+
return int(text)
|
|
699
|
+
if ROMAN_RE.fullmatch(text):
|
|
700
|
+
return roman_value(text)
|
|
701
|
+
return None
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
class _Blocks:
|
|
705
|
+
"""Minimal block extractor over html.parser; tracks the innermost block tag
|
|
706
|
+
so each emitted (tag, text) pair is one rendered block in reading order.
|
|
707
|
+
|
|
708
|
+
img_before[i] is True when an image (img/svg) appeared between the last
|
|
709
|
+
text before blocks[i] and blocks[i]'s own first text. Used by the ocr
|
|
710
|
+
analyzer to clear paragraphs interrupted by a rendered figure (an inline
|
|
711
|
+
formula or card diagram reads as a mid-sentence split otherwise).
|
|
712
|
+
in_quote[i] is True when blocks[i] was emitted inside a <blockquote>
|
|
713
|
+
(a display quotation legitimately starts and ends mid-sentence). Both are
|
|
714
|
+
ocr-only; pagenumbers ignores them."""
|
|
715
|
+
|
|
716
|
+
def __init__(self):
|
|
717
|
+
from html.parser import HTMLParser
|
|
718
|
+
|
|
719
|
+
outer = self
|
|
720
|
+
|
|
721
|
+
class _P(HTMLParser):
|
|
722
|
+
def __init__(self):
|
|
723
|
+
super().__init__(convert_charrefs=True)
|
|
724
|
+
self.stack: list[str] = []
|
|
725
|
+
self.buf: list[str] = []
|
|
726
|
+
self.img_gap = False # image seen since the last text content
|
|
727
|
+
self.block_img = False # img_gap captured at this block's first text
|
|
728
|
+
self.has_text = False
|
|
729
|
+
|
|
730
|
+
def handle_starttag(self, tag, attrs):
|
|
731
|
+
del attrs
|
|
732
|
+
if tag in ("img", "image", "svg"):
|
|
733
|
+
self.img_gap = True
|
|
734
|
+
if tag in BLOCK_TAGS:
|
|
735
|
+
outer._flush(self)
|
|
736
|
+
self.stack.append(tag)
|
|
737
|
+
|
|
738
|
+
def handle_endtag(self, tag):
|
|
739
|
+
if tag in BLOCK_TAGS:
|
|
740
|
+
outer._flush(self)
|
|
741
|
+
if self.stack:
|
|
742
|
+
self.stack.pop()
|
|
743
|
+
|
|
744
|
+
def handle_data(self, data):
|
|
745
|
+
if data.strip():
|
|
746
|
+
if not self.has_text:
|
|
747
|
+
self.block_img = self.img_gap
|
|
748
|
+
self.has_text = True
|
|
749
|
+
self.img_gap = False
|
|
750
|
+
self.buf.append(data)
|
|
751
|
+
|
|
752
|
+
self.blocks: list[tuple[str, str]] = []
|
|
753
|
+
self.img_before: list[bool] = []
|
|
754
|
+
self.in_quote: list[bool] = []
|
|
755
|
+
self._parser = _P()
|
|
756
|
+
|
|
757
|
+
def _flush(self, parser):
|
|
758
|
+
text = "".join(parser.buf).strip()
|
|
759
|
+
parser.buf.clear()
|
|
760
|
+
if text:
|
|
761
|
+
tag = parser.stack[-1] if parser.stack else "?"
|
|
762
|
+
self.blocks.append((tag, text))
|
|
763
|
+
self.img_before.append(parser.block_img)
|
|
764
|
+
self.in_quote.append("blockquote" in parser.stack)
|
|
765
|
+
parser.has_text = False
|
|
766
|
+
parser.block_img = False
|
|
767
|
+
|
|
768
|
+
def feed(self, html: str):
|
|
769
|
+
self._parser.feed(html)
|
|
770
|
+
self._parser.close()
|
|
771
|
+
self._flush(self._parser)
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def analyze_pagenumbers(book: Book) -> dict:
|
|
775
|
+
"""Score baked-in page numbers over the (pre-read) spine, skipping nav."""
|
|
776
|
+
blocks: list[tuple[str, str]] = []
|
|
777
|
+
for doc in book.spine:
|
|
778
|
+
if book.nav and doc == book.nav:
|
|
779
|
+
continue
|
|
780
|
+
parser = _Blocks()
|
|
781
|
+
try:
|
|
782
|
+
parser.feed(book.docs.get(doc, ""))
|
|
783
|
+
except Exception:
|
|
784
|
+
continue
|
|
785
|
+
li = sum(1 for t, _ in parser.blocks if t == "li")
|
|
786
|
+
# a doc that is mostly <li> is a TOC / page-list, not body text
|
|
787
|
+
if parser.blocks and li / len(parser.blocks) > 0.5:
|
|
788
|
+
continue
|
|
789
|
+
blocks.extend(parser.blocks)
|
|
790
|
+
|
|
791
|
+
text_len = sum(len(t) for _, t in blocks) or 1
|
|
792
|
+
|
|
793
|
+
# Running headers / footers / watermarks: short blocks repeated many times.
|
|
794
|
+
freq = Counter(
|
|
795
|
+
t
|
|
796
|
+
for t in (b[1] for b in blocks)
|
|
797
|
+
if len(t) <= RUNHEAD_MAX_LEN and number_value(t) is None
|
|
798
|
+
)
|
|
799
|
+
runheads = {t for t, c in freq.items() if c >= RUNHEAD_MIN_REPEAT}
|
|
800
|
+
|
|
801
|
+
hits: list[tuple[int, int]] = [] # (char offset, value)
|
|
802
|
+
examples: list[dict] = []
|
|
803
|
+
offset = 0
|
|
804
|
+
for i, (tag, text) in enumerate(blocks):
|
|
805
|
+
if tag in ("p", "div"):
|
|
806
|
+
v = number_value(text)
|
|
807
|
+
# 1500-2099 are almost always years (chronologies, dated chapters),
|
|
808
|
+
# not page numbers; a real page count rarely reaches them.
|
|
809
|
+
if v is not None and not (1500 <= v <= 2099):
|
|
810
|
+
ptag, ptxt = blocks[i - 1] if i > 0 else ("", "")
|
|
811
|
+
ntag, ntxt = blocks[i + 1] if i + 1 < len(blocks) else ("", "")
|
|
812
|
+
prose_prev = ptag in ("p", "div") and len(ptxt) > PROSE_MIN
|
|
813
|
+
prose_next = ntag in ("p", "div") and len(ntxt) > PROSE_MIN
|
|
814
|
+
if prose_prev or prose_next:
|
|
815
|
+
prev_runhead = ptxt in runheads
|
|
816
|
+
next_runhead = ntxt in runheads
|
|
817
|
+
word_split = (
|
|
818
|
+
bool(ptxt) and ptxt[-1] == "-" and ptxt[-2:-1].isalpha()
|
|
819
|
+
)
|
|
820
|
+
lower_cont = bool(ntxt) and ntxt[0].islower()
|
|
821
|
+
# the previous body paragraph is unfinished (ends mid-word /
|
|
822
|
+
# mid-clause), so the number is wedged into a live sentence.
|
|
823
|
+
prev_unfinished = prose_prev and (
|
|
824
|
+
ptxt[-1].islower() or ptxt[-1] == ","
|
|
825
|
+
)
|
|
826
|
+
baked = (
|
|
827
|
+
word_split
|
|
828
|
+
or lower_cont
|
|
829
|
+
or (prev_unfinished and (prose_next or next_runhead))
|
|
830
|
+
or (prev_runhead and next_runhead)
|
|
831
|
+
)
|
|
832
|
+
if baked:
|
|
833
|
+
hits.append((offset, v))
|
|
834
|
+
if len(examples) < 8:
|
|
835
|
+
examples.append(
|
|
836
|
+
{"v": text, "prev": ptxt[-60:], "next": ntxt[:60]}
|
|
837
|
+
)
|
|
838
|
+
offset += len(text)
|
|
839
|
+
|
|
840
|
+
vals = [v for _, v in hits]
|
|
841
|
+
if hits:
|
|
842
|
+
span = (hits[-1][0] - hits[0][0]) / text_len
|
|
843
|
+
else:
|
|
844
|
+
span = 0.0
|
|
845
|
+
run = best = 1 if vals else 0
|
|
846
|
+
for i in range(1, len(vals)):
|
|
847
|
+
if vals[i] - vals[i - 1] in (1, 2):
|
|
848
|
+
best += 1
|
|
849
|
+
run = max(run, best)
|
|
850
|
+
else:
|
|
851
|
+
best = 1
|
|
852
|
+
return {
|
|
853
|
+
"n_hits": len(hits),
|
|
854
|
+
"span": span,
|
|
855
|
+
"run": run,
|
|
856
|
+
"watermark": any(
|
|
857
|
+
re.search(r"download|boykma|\.com\b", h, re.IGNORECASE) for h in runheads
|
|
858
|
+
),
|
|
859
|
+
"examples": examples,
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
def is_defective(r: dict) -> bool:
|
|
864
|
+
return (
|
|
865
|
+
r["n_hits"] >= MIN_BAKED_HITS and r["span"] >= MIN_SPAN and r["run"] >= MIN_RUN
|
|
866
|
+
)
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
def scan_pagenumbers(path: Path) -> dict:
|
|
870
|
+
return analyze_pagenumbers(load_book(path))
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
# ----------------------------------------------------------------------------
|
|
874
|
+
# Analyzer: emptytext (empty / no-body-text stubs)
|
|
875
|
+
# ----------------------------------------------------------------------------
|
|
876
|
+
|
|
877
|
+
DEFAULT_MIN_CHARS = 2000 # at or below this: EMPTY (real defect)
|
|
878
|
+
DEFAULT_THIN_CHARS = 20000 # below this: THIN (advisory review)
|
|
879
|
+
# Monolithic-document floor (phase-1 skill: "roughly 300-500k" chars is where
|
|
880
|
+
# readers start refusing). A single content doc at or above this flags.
|
|
881
|
+
DEFAULT_MAX_DOC_CHARS = 300_000
|
|
882
|
+
|
|
883
|
+
IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg")
|
|
884
|
+
BOOKMATE_MARKERS = ("bookmate.css", "calibre_bookmarks.txt")
|
|
885
|
+
|
|
886
|
+
# Partial / placeholder exports: a DRM-locked or sample export leaves most
|
|
887
|
+
# chapters as the same tiny "content unavailable" placeholder, so the book
|
|
888
|
+
# validates and clears the total-char floor on the strength of one or two real
|
|
889
|
+
# chapters (the whole-book char count is fooled). Detect by a known signature,
|
|
890
|
+
# or by the same short stub repeated across a large fraction of the spine.
|
|
891
|
+
# Seen cases: BookShout ("something went wrong loading... bookshout.com").
|
|
892
|
+
_PLACEHOLDER_SIG = re.compile(
|
|
893
|
+
r"bookshout|something went wrong loading|failed to load|"
|
|
894
|
+
r"(content|page|book)\s+(is\s+)?(not available|unavailable|could not be loaded)",
|
|
895
|
+
re.IGNORECASE,
|
|
896
|
+
)
|
|
897
|
+
PLACEHOLDER_STUB_MIN = 12 # ignore blank / trivial spine docs
|
|
898
|
+
PLACEHOLDER_STUB_MAX = 600 # a placeholder stub is short
|
|
899
|
+
PLACEHOLDER_MIN_REPEAT = 3 # the same stub across at least this many spine docs
|
|
900
|
+
PLACEHOLDER_MIN_FRAC = 0.30 # ...and at least this fraction of the spine
|
|
901
|
+
|
|
902
|
+
_SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b.*?</\1>", re.IGNORECASE | re.DOTALL)
|
|
903
|
+
_TAG_RE = re.compile(r"<[^>]+>")
|
|
904
|
+
_WS_RE = re.compile(r"\s+")
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
def _visible_text(html: str) -> str:
|
|
908
|
+
"""Rendered text: drop script/style, strip tags, decode entities, collapse
|
|
909
|
+
whitespace."""
|
|
910
|
+
from html import unescape
|
|
911
|
+
|
|
912
|
+
html = _SCRIPT_STYLE_RE.sub(" ", html)
|
|
913
|
+
text = _TAG_RE.sub(" ", html)
|
|
914
|
+
text = unescape(text)
|
|
915
|
+
return _WS_RE.sub(" ", text).strip()
|
|
916
|
+
|
|
917
|
+
|
|
918
|
+
def _visible_chars(html: str) -> int:
|
|
919
|
+
return len(_visible_text(html))
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
def analyze_emptytext(book: Book) -> dict:
|
|
923
|
+
"""Count visible text across the (pre-read) spine and gather triage signals."""
|
|
924
|
+
texts = book.visible_texts()
|
|
925
|
+
chars = sum(len(t) for t in texts)
|
|
926
|
+
images = sum(1 for n in book.names if n.lower().endswith(IMAGE_EXTS))
|
|
927
|
+
bookmate = any(any(m in n.lower() for m in BOOKMATE_MARKERS) for n in book.names)
|
|
928
|
+
|
|
929
|
+
# Partial / placeholder export: a known DRM signature anywhere, or the same
|
|
930
|
+
# short stub repeated across a large fraction of the spine (most chapters
|
|
931
|
+
# replaced by an identical "content unavailable" notice). Blank docs are
|
|
932
|
+
# excluded by PLACEHOLDER_STUB_MIN so well-made books full of small section
|
|
933
|
+
# dividers (each with distinct text) do not trip it.
|
|
934
|
+
sig = any(_PLACEHOLDER_SIG.search(t) for t in texts)
|
|
935
|
+
stubs = Counter(
|
|
936
|
+
t for t in texts if PLACEHOLDER_STUB_MIN <= len(t) <= PLACEHOLDER_STUB_MAX
|
|
937
|
+
)
|
|
938
|
+
stub_n = stubs.most_common(1)[0][1] if stubs else 0
|
|
939
|
+
spine_n = max(1, len(book.spine))
|
|
940
|
+
repeated = (
|
|
941
|
+
stub_n >= PLACEHOLDER_MIN_REPEAT and stub_n / spine_n >= PLACEHOLDER_MIN_FRAC
|
|
942
|
+
)
|
|
943
|
+
return {
|
|
944
|
+
"chars": chars,
|
|
945
|
+
"spine_len": len(book.spine),
|
|
946
|
+
"images": images,
|
|
947
|
+
"bookmate": bookmate,
|
|
948
|
+
"placeholder": bool(sig or repeated),
|
|
949
|
+
"placeholder_sig": sig,
|
|
950
|
+
"stub_docs": stub_n,
|
|
951
|
+
"corrupt_n": len(book.corrupt),
|
|
952
|
+
"corrupt_first": book.corrupt[0] if book.corrupt else "",
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def classify(r: dict, min_chars: int, thin_chars: int) -> str:
|
|
957
|
+
# A corrupt archive entry reads as empty text; reporting EMPTY here would
|
|
958
|
+
# be the right alarm for the wrong disease. The corruption verdict owns it.
|
|
959
|
+
if r.get("corrupt_n"):
|
|
960
|
+
return "CORRUPT"
|
|
961
|
+
if r["chars"] <= min_chars:
|
|
962
|
+
return "EMPTY"
|
|
963
|
+
if r.get("placeholder"):
|
|
964
|
+
return "PARTIAL"
|
|
965
|
+
if r["chars"] < thin_chars:
|
|
966
|
+
return "THIN"
|
|
967
|
+
return "OK"
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def _empty_detail(r: dict) -> str:
|
|
971
|
+
bits = [f"{r['chars']} chars", f"spine {r['spine_len']}", f"{r['images']} images"]
|
|
972
|
+
if r.get("corrupt_n"):
|
|
973
|
+
bits.append(f"corrupt:{r['corrupt_n']} (first: {r['corrupt_first']})")
|
|
974
|
+
if r["bookmate"]:
|
|
975
|
+
bits.append("bookmate")
|
|
976
|
+
if r.get("placeholder"):
|
|
977
|
+
if r.get("placeholder_sig"):
|
|
978
|
+
bits.append("partial: DRM/placeholder signature")
|
|
979
|
+
else:
|
|
980
|
+
bits.append(
|
|
981
|
+
f"partial: {r['stub_docs']}/{r['spine_len']} identical stub docs"
|
|
982
|
+
)
|
|
983
|
+
return ", ".join(bits)
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
def scan_emptytext(path: Path) -> dict:
|
|
987
|
+
return analyze_emptytext(load_book(path))
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
# ----------------------------------------------------------------------------
|
|
991
|
+
# Analyzer: monolithic (one spine doc that is far too large)
|
|
992
|
+
# ----------------------------------------------------------------------------
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def analyze_monolithic(book: Book) -> dict:
|
|
996
|
+
"""Find the largest single content document, in characters.
|
|
997
|
+
|
|
998
|
+
Monolithic documents are invisible to emptytext (whole-book volume, not
|
|
999
|
+
per-doc shape) and to epubcheck (which never sees renderer memory limits):
|
|
1000
|
+
a "clean" 30M-char dictionary that will not render past a point on real
|
|
1001
|
+
readers. Reuses the shared visible_texts() cache, so this adds no second
|
|
1002
|
+
decompression pass.
|
|
1003
|
+
"""
|
|
1004
|
+
texts = book.visible_texts()
|
|
1005
|
+
worst_i = max(range(len(texts)), key=lambda i: len(texts[i]), default=None)
|
|
1006
|
+
return {
|
|
1007
|
+
"max_doc_chars": len(texts[worst_i]) if worst_i is not None else 0,
|
|
1008
|
+
"worst_doc": book.spine[worst_i] if worst_i is not None else "",
|
|
1009
|
+
"spine_len": len(book.spine),
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
|
|
1013
|
+
def is_monolithic(r: dict, max_doc_chars: int) -> bool:
|
|
1014
|
+
"""Flag only at or above the threshold; high-but-under stays silent
|
|
1015
|
+
(advisory silence, like THIN)."""
|
|
1016
|
+
return r["max_doc_chars"] >= max_doc_chars
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def scan_monolithic(path: Path) -> dict:
|
|
1020
|
+
return analyze_monolithic(load_book(path))
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
# ----------------------------------------------------------------------------
|
|
1024
|
+
# Analyzer: archive corruption (always on; owns the "empty body" story)
|
|
1025
|
+
# ----------------------------------------------------------------------------
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def analyze_corrupt(book: Book) -> dict:
|
|
1029
|
+
"""Archive entries whose full read failed (bad CRC, truncated stream).
|
|
1030
|
+
|
|
1031
|
+
Reported as its own verdict in every mode — a corrupt entry decompresses
|
|
1032
|
+
to nothing, and letting emptytext call that EMPTY mislabels a damaged
|
|
1033
|
+
archive as a content-less stub (the phase-1 re-source advice that follows
|
|
1034
|
+
from EMPTY would then aim at the wrong disease).
|
|
1035
|
+
"""
|
|
1036
|
+
return {"n": len(book.corrupt), "first": book.corrupt[0] if book.corrupt else ""}
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
def _corrupt_verdict(r: dict) -> tuple[bool, str, list[str]]:
|
|
1040
|
+
return (
|
|
1041
|
+
True,
|
|
1042
|
+
"CORRUPT",
|
|
1043
|
+
[f"corrupt:{r['n']} (first: {r['first']}) — damaged archive; re-source"],
|
|
1044
|
+
)
|
|
1045
|
+
|
|
1046
|
+
|
|
1047
|
+
# ----------------------------------------------------------------------------
|
|
1048
|
+
# Analyzer: spine integrity (ToC bloat vs a true fragment)
|
|
1049
|
+
# ----------------------------------------------------------------------------
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def spine_integrity(book: Book) -> dict:
|
|
1053
|
+
"""Classify absent ToC/NCX targets.
|
|
1054
|
+
|
|
1055
|
+
The official Wandering Inn builds ship series-wide ToC manifests (~750
|
|
1056
|
+
references vs 14-19 real content docs): `convention` when the absent
|
|
1057
|
+
count is ~= the reference count and the present docs' chapter span is
|
|
1058
|
+
consecutive — reported, never flagged. A broken span means the book is
|
|
1059
|
+
missing part of itself: `fragment`, flagged. Unnumbered or absent==0
|
|
1060
|
+
cases are `ok`/`unknown` (silent or advisory).
|
|
1061
|
+
"""
|
|
1062
|
+
absent, refs = book.toc_absent, book.toc_refs
|
|
1063
|
+
if absent == 0:
|
|
1064
|
+
return {"class": "ok", "refs": refs, "absent": 0}
|
|
1065
|
+
nums = []
|
|
1066
|
+
for doc in book.spine:
|
|
1067
|
+
found = re.findall(r"\d+", doc.rsplit("/", 1)[-1])
|
|
1068
|
+
if not found:
|
|
1069
|
+
return {"class": "unknown", "refs": refs, "absent": absent}
|
|
1070
|
+
nums.append(int(found[-1]))
|
|
1071
|
+
nums.sort()
|
|
1072
|
+
span = (
|
|
1073
|
+
"consecutive" if nums == list(range(nums[0], nums[0] + len(nums))) else "broken"
|
|
1074
|
+
)
|
|
1075
|
+
if refs >= 20 and absent >= refs * 0.9 and span == "consecutive":
|
|
1076
|
+
return {"class": "convention", "refs": refs, "absent": absent}
|
|
1077
|
+
if span == "broken":
|
|
1078
|
+
return {"class": "fragment", "refs": refs, "absent": absent}
|
|
1079
|
+
return {"class": "unknown", "refs": refs, "absent": absent}
|
|
1080
|
+
|
|
1081
|
+
|
|
1082
|
+
def _spine_verdict(r: dict) -> tuple[bool, str, list[str]]:
|
|
1083
|
+
if r["class"] == "fragment":
|
|
1084
|
+
return (
|
|
1085
|
+
True,
|
|
1086
|
+
"FRAGMENT",
|
|
1087
|
+
[f"toc {r['refs']} refs, {r['absent']} absent — span broken"],
|
|
1088
|
+
)
|
|
1089
|
+
if r["class"] == "convention":
|
|
1090
|
+
return (
|
|
1091
|
+
False,
|
|
1092
|
+
"ADVISORY",
|
|
1093
|
+
[
|
|
1094
|
+
f"toc {r['refs']} refs, {r['absent']} absent (series-wide manifest convention)"
|
|
1095
|
+
],
|
|
1096
|
+
)
|
|
1097
|
+
return (
|
|
1098
|
+
False,
|
|
1099
|
+
"ADVISORY",
|
|
1100
|
+
[f"toc {r['refs']} refs, {r['absent']} absent (unjudgeable span)"],
|
|
1101
|
+
)
|
|
1102
|
+
|
|
1103
|
+
|
|
1104
|
+
# ----------------------------------------------------------------------------
|
|
1105
|
+
# Analyzer: ocr (OCR/conversion-damaged prose)
|
|
1106
|
+
# ----------------------------------------------------------------------------
|
|
1107
|
+
|
|
1108
|
+
# Tuning, validated by hand against a 4,605-EPUB library: every book passing
|
|
1109
|
+
# the joint gate was inspected via its split examples. At these values the
|
|
1110
|
+
# detector flagged 105 books; 104 were confirmed damage (line-wrap and
|
|
1111
|
+
# page-break splits, double-spaced OCR text, running headers wedged into
|
|
1112
|
+
# sentences), with one borderline residue (a book whose display quotes are
|
|
1113
|
+
# publisher-styled plain <p>s, indistinguishable from damage without CSS).
|
|
1114
|
+
# The motivating case: a damaged Jingo EPUB measured 80 mid-sentence splits
|
|
1115
|
+
# where a clean edition of the same text measured 0.
|
|
1116
|
+
#
|
|
1117
|
+
# Known misses, by design: damage whose signature is word TRUNCATION ("which
|
|
1118
|
+
# bel[ong] to western Spain") or whitespace corruption rather than paragraph
|
|
1119
|
+
# splitting, and low-grade split damage that lands in the same func_frac band
|
|
1120
|
+
# as literary stream-of-consciousness (Gulag Archipelago at 0.243 vs. The
|
|
1121
|
+
# Sound and the Fury at 0.242: no threshold separates them).
|
|
1122
|
+
OCR_MIN_PARAS = 50 # below this the rate is noise, not a signal
|
|
1123
|
+
OCR_MIN_SPLITS = 10 # absolute floor; a handful of splits is coincidence
|
|
1124
|
+
OCR_FLAG_RATE = 0.010 # splits per prose paragraph; FLAG at or above
|
|
1125
|
+
OCR_FUNC_MIN = 0.25 # see func_frac below
|
|
1126
|
+
|
|
1127
|
+
# A split only counts when one side is a substantial prose block (reuses the
|
|
1128
|
+
# pagenumbers notion of prose), so verse and dialogue beats -- short lines that
|
|
1129
|
+
# legitimately end unpunctuated and start lowercase -- do not accumulate.
|
|
1130
|
+
_OCR_PROSE_MIN = PROSE_MIN
|
|
1131
|
+
|
|
1132
|
+
# Residual false-positive guards, each from a class found in the validation
|
|
1133
|
+
# sweep. A back-of-book index rendered as <p> blocks reads as a wall of
|
|
1134
|
+
# unpunctuated lowercase runs ("See also" is its near-universal signature).
|
|
1135
|
+
# An epistolary sign-off is a dangling sub-40-char fragment with no terminal
|
|
1136
|
+
# punctuation ("in which hope I rest," / "respected sir,"). A display equation
|
|
1137
|
+
# set as text is mostly non-alphabetic ("the equation" / "y2 + y = x3 - x ?")
|
|
1138
|
+
# and renders fine on its own line.
|
|
1139
|
+
_OCR_INDEX_RE = re.compile(r"\bSee also\b")
|
|
1140
|
+
_OCR_INDEX_MIN_BLOCKS = 3
|
|
1141
|
+
_OCR_TAIL_MIN = 40
|
|
1142
|
+
_OCR_TERMINALS = ".!?\"'’”…)"
|
|
1143
|
+
_OCR_ALPHA_MIN = 0.5
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def _alpha_density(text: str) -> float:
|
|
1147
|
+
chars = [c for c in text if not c.isspace()]
|
|
1148
|
+
if not chars:
|
|
1149
|
+
return 0.0
|
|
1150
|
+
return sum(c.isalpha() for c in chars) / len(chars)
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
# The style-vs-damage discriminator. Deliberately unpunctuated literary prose
|
|
1154
|
+
# (Fosse's Septology, Evaristo's "Girl, Woman, Other", Kingsnorth's "The Wake")
|
|
1155
|
+
# racks up enormous split rates, but its paragraphs end at CLAUSE boundaries;
|
|
1156
|
+
# conversion damage splits at line-wrap/page-break positions, so the fragment
|
|
1157
|
+
# ends on a function word ("sat the disembodied" / "heads who were..."). On the
|
|
1158
|
+
# reference library, style books measured func_frac <= 0.11 and every hand-
|
|
1159
|
+
# confirmed damage case >= 0.26, so 0.25 separates cleanly. A small closed
|
|
1160
|
+
# function-word set, same spirit as the content analyzer's stopword votes --
|
|
1161
|
+
# not a dictionary.
|
|
1162
|
+
OCR_FUNC_WORDS = frozenset(
|
|
1163
|
+
[
|
|
1164
|
+
"the",
|
|
1165
|
+
"a",
|
|
1166
|
+
"an",
|
|
1167
|
+
"and",
|
|
1168
|
+
"or",
|
|
1169
|
+
"but",
|
|
1170
|
+
"of",
|
|
1171
|
+
"to",
|
|
1172
|
+
"in",
|
|
1173
|
+
"on",
|
|
1174
|
+
"at",
|
|
1175
|
+
"by",
|
|
1176
|
+
"with",
|
|
1177
|
+
"for",
|
|
1178
|
+
"from",
|
|
1179
|
+
"as",
|
|
1180
|
+
"that",
|
|
1181
|
+
"this",
|
|
1182
|
+
"these",
|
|
1183
|
+
"those",
|
|
1184
|
+
"his",
|
|
1185
|
+
"her",
|
|
1186
|
+
"their",
|
|
1187
|
+
"its",
|
|
1188
|
+
"my",
|
|
1189
|
+
"your",
|
|
1190
|
+
"our",
|
|
1191
|
+
"was",
|
|
1192
|
+
"were",
|
|
1193
|
+
"is",
|
|
1194
|
+
"are",
|
|
1195
|
+
"be",
|
|
1196
|
+
"been",
|
|
1197
|
+
"being",
|
|
1198
|
+
"had",
|
|
1199
|
+
"has",
|
|
1200
|
+
"have",
|
|
1201
|
+
"he",
|
|
1202
|
+
"she",
|
|
1203
|
+
"they",
|
|
1204
|
+
"it",
|
|
1205
|
+
"we",
|
|
1206
|
+
"you",
|
|
1207
|
+
"i",
|
|
1208
|
+
"not",
|
|
1209
|
+
"no",
|
|
1210
|
+
"so",
|
|
1211
|
+
"if",
|
|
1212
|
+
"when",
|
|
1213
|
+
"than",
|
|
1214
|
+
"then",
|
|
1215
|
+
"who",
|
|
1216
|
+
"whom",
|
|
1217
|
+
"which",
|
|
1218
|
+
"into",
|
|
1219
|
+
"onto",
|
|
1220
|
+
"over",
|
|
1221
|
+
"under",
|
|
1222
|
+
"between",
|
|
1223
|
+
"through",
|
|
1224
|
+
"during",
|
|
1225
|
+
"before",
|
|
1226
|
+
"after",
|
|
1227
|
+
"above",
|
|
1228
|
+
"below",
|
|
1229
|
+
"up",
|
|
1230
|
+
"down",
|
|
1231
|
+
"out",
|
|
1232
|
+
"off",
|
|
1233
|
+
"very",
|
|
1234
|
+
"more",
|
|
1235
|
+
"most",
|
|
1236
|
+
"some",
|
|
1237
|
+
"any",
|
|
1238
|
+
"each",
|
|
1239
|
+
"every",
|
|
1240
|
+
"either",
|
|
1241
|
+
"neither",
|
|
1242
|
+
]
|
|
1243
|
+
)
|
|
1244
|
+
|
|
1245
|
+
# En-dash embedded inside a word: OCR reads a hyphen as U+2013 ("bottom–feedin'").
|
|
1246
|
+
EN_DASH_WORD_RE = re.compile(r"[A-Za-z]–[A-Za-z]")
|
|
1247
|
+
# Doubled opening quote: an opening single quote re-recognized before a
|
|
1248
|
+
# dialogue contraction ("' 'Course"). Straight or curly. The first quote must
|
|
1249
|
+
# follow whitespace: a closing quote hugs the word before it, so this stays
|
|
1250
|
+
# blind to legitimate close-then-open sequences ("...said.' 'No...") that
|
|
1251
|
+
# single-quote-dialogue books produce constantly.
|
|
1252
|
+
DOUBLED_QUOTE_RE = re.compile(r"(?<=\s)['‘]\s+['‘’]\w")
|
|
1253
|
+
# A space-stripped proper-noun compound: CamelCase with 2+ humps
|
|
1254
|
+
# ("AnkhMorpork"); only damage when the hyphenated form also appears.
|
|
1255
|
+
_CAMEL_RE = re.compile(r"\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\b")
|
|
1256
|
+
_HUMP_RE = re.compile(r"(?<=[a-z])(?=[A-Z])")
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
def analyze_brokentags(book: Book) -> dict:
|
|
1260
|
+
import re
|
|
1261
|
+
|
|
1262
|
+
hits = []
|
|
1263
|
+
pattern = re.compile(
|
|
1264
|
+
r"(?<!<)(?<!<)/[a-zA-Z]+>|</(?:p|div|span|h[1-6]|i|em|b|strong)>",
|
|
1265
|
+
re.IGNORECASE,
|
|
1266
|
+
)
|
|
1267
|
+
for doc in book.spine:
|
|
1268
|
+
text = book.docs.get(doc, "")
|
|
1269
|
+
for match in pattern.finditer(text):
|
|
1270
|
+
hits.append(f"{doc}: {match.group(0)}")
|
|
1271
|
+
if not hits:
|
|
1272
|
+
return {}
|
|
1273
|
+
return {"hits": hits, "summary": f"{len(hits)} broken tags found"}
|
|
1274
|
+
|
|
1275
|
+
|
|
1276
|
+
def analyze_ocr(book: Book) -> dict:
|
|
1277
|
+
"""Score OCR/conversion damage over the (pre-read) spine, skipping nav.
|
|
1278
|
+
|
|
1279
|
+
Primary signal: mid-sentence paragraph splits -- a prose block ends without
|
|
1280
|
+
terminal punctuation (last char a lowercase letter or comma) and the next
|
|
1281
|
+
prose block starts lowercase. Splits are only paired within one spine doc;
|
|
1282
|
+
chapter-file boundaries never produce a pair, and a pair interrupted by a
|
|
1283
|
+
rendered image (inline formula, card diagram) is cleared."""
|
|
1284
|
+
paras = 0
|
|
1285
|
+
splits = 0
|
|
1286
|
+
func_splits = 0
|
|
1287
|
+
examples: list[dict] = []
|
|
1288
|
+
for doc in book.spine:
|
|
1289
|
+
if book.nav and doc == book.nav:
|
|
1290
|
+
continue
|
|
1291
|
+
parser = _Blocks()
|
|
1292
|
+
try:
|
|
1293
|
+
parser.feed(book.docs.get(doc, ""))
|
|
1294
|
+
except Exception:
|
|
1295
|
+
continue
|
|
1296
|
+
li = sum(1 for t, _ in parser.blocks if t == "li")
|
|
1297
|
+
# a doc that is mostly <li> is a TOC / page-list, not body text
|
|
1298
|
+
if parser.blocks and li / len(parser.blocks) > 0.5:
|
|
1299
|
+
continue
|
|
1300
|
+
# a back-of-book index rendered as <p> blocks is not body text either
|
|
1301
|
+
if (
|
|
1302
|
+
sum(1 for _, t in parser.blocks if _OCR_INDEX_RE.search(t))
|
|
1303
|
+
>= _OCR_INDEX_MIN_BLOCKS
|
|
1304
|
+
):
|
|
1305
|
+
continue
|
|
1306
|
+
paras += sum(1 for t, _ in parser.blocks if t in ("p", "div"))
|
|
1307
|
+
# pairs must be adjacent in reading order: an intervening heading /
|
|
1308
|
+
# scene-break block means a boundary, not a mid-sentence split
|
|
1309
|
+
for i in range(len(parser.blocks) - 1):
|
|
1310
|
+
ptag, prev = parser.blocks[i]
|
|
1311
|
+
ntag, nxt = parser.blocks[i + 1]
|
|
1312
|
+
if ptag not in ("p", "div") or ntag not in ("p", "div"):
|
|
1313
|
+
continue
|
|
1314
|
+
if parser.img_before[i + 1]:
|
|
1315
|
+
continue
|
|
1316
|
+
# a display quotation starts (and its resumption ends) mid-sentence
|
|
1317
|
+
# by design; never pair into or out of a <blockquote>
|
|
1318
|
+
if parser.in_quote[i] or parser.in_quote[i + 1]:
|
|
1319
|
+
continue
|
|
1320
|
+
# a dangling short unterminated fragment is a sign-off, not a
|
|
1321
|
+
# sentence remnant; a mostly-non-alphabetic fragment is display math
|
|
1322
|
+
if len(nxt) < _OCR_TAIL_MIN and nxt[-1] not in _OCR_TERMINALS:
|
|
1323
|
+
continue
|
|
1324
|
+
if min(_alpha_density(prev), _alpha_density(nxt)) < _OCR_ALPHA_MIN:
|
|
1325
|
+
continue
|
|
1326
|
+
unfinished = prev[-1].islower() or prev[-1] == ","
|
|
1327
|
+
lower_start = nxt[0].islower()
|
|
1328
|
+
substantial = len(prev) > _OCR_PROSE_MIN or len(nxt) > _OCR_PROSE_MIN
|
|
1329
|
+
if unfinished and lower_start and substantial:
|
|
1330
|
+
words = prev.rstrip(",").split()
|
|
1331
|
+
last = words[-1].lower() if words else ""
|
|
1332
|
+
# "Lordat noticed that" / lowercase block is the block-
|
|
1333
|
+
# quotation idiom of academic prose, not damage: an attribution
|
|
1334
|
+
# ending on "that" introduces a quote set as its own block,
|
|
1335
|
+
# which rightly starts lowercase
|
|
1336
|
+
if last == "that":
|
|
1337
|
+
continue
|
|
1338
|
+
splits += 1
|
|
1339
|
+
if last in OCR_FUNC_WORDS:
|
|
1340
|
+
func_splits += 1
|
|
1341
|
+
if len(examples) < 8:
|
|
1342
|
+
examples.append({"prev": prev[-60:], "next": nxt[:60]})
|
|
1343
|
+
|
|
1344
|
+
# Side signals over the visible text (dictionary-free by design).
|
|
1345
|
+
text = " ".join(book.visible_texts())
|
|
1346
|
+
glued: list[str] = []
|
|
1347
|
+
for w in set(_CAMEL_RE.findall(text)):
|
|
1348
|
+
hyphenated = _HUMP_RE.sub("-", w)
|
|
1349
|
+
if hyphenated in text:
|
|
1350
|
+
glued.append(f"{w}~{hyphenated}")
|
|
1351
|
+
return {
|
|
1352
|
+
"paras": paras,
|
|
1353
|
+
"splits": splits,
|
|
1354
|
+
"split_rate": splits / paras if paras else 0.0,
|
|
1355
|
+
"func_frac": func_splits / splits if splits else 0.0,
|
|
1356
|
+
"en_dash_words": len(EN_DASH_WORD_RE.findall(text)),
|
|
1357
|
+
"doubled_quotes": len(DOUBLED_QUOTE_RE.findall(text)),
|
|
1358
|
+
"glued": sorted(glued)[:8],
|
|
1359
|
+
"examples": examples,
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
|
|
1363
|
+
def is_ocr_damaged(r: dict) -> bool:
|
|
1364
|
+
return (
|
|
1365
|
+
r["paras"] >= OCR_MIN_PARAS
|
|
1366
|
+
and r["splits"] >= OCR_MIN_SPLITS
|
|
1367
|
+
and r["split_rate"] >= OCR_FLAG_RATE
|
|
1368
|
+
and r["func_frac"] >= OCR_FUNC_MIN
|
|
1369
|
+
)
|
|
1370
|
+
|
|
1371
|
+
|
|
1372
|
+
def _ocr_detail(r: dict) -> str:
|
|
1373
|
+
bits = [
|
|
1374
|
+
f"{r['splits']} mid-sentence splits / {r['paras']} paragraphs "
|
|
1375
|
+
f"({r['split_rate'] * 100:.1f}%, {r['func_frac'] * 100:.0f}% on function words)"
|
|
1376
|
+
]
|
|
1377
|
+
if r["en_dash_words"]:
|
|
1378
|
+
bits.append(f"en-dash-in-word x{r['en_dash_words']}")
|
|
1379
|
+
if r["doubled_quotes"]:
|
|
1380
|
+
bits.append(f"doubled quotes x{r['doubled_quotes']}")
|
|
1381
|
+
if r["glued"]:
|
|
1382
|
+
bits.append("glued nouns: " + ", ".join(r["glued"][:3]))
|
|
1383
|
+
return ", ".join(bits)
|
|
1384
|
+
|
|
1385
|
+
|
|
1386
|
+
def scan_ocr(path: Path) -> dict:
|
|
1387
|
+
return analyze_ocr(load_book(path))
|
|
1388
|
+
|
|
1389
|
+
|
|
1390
|
+
# ----------------------------------------------------------------------------
|
|
1391
|
+
# Per-analyzer reporting (library mode)
|
|
1392
|
+
# ----------------------------------------------------------------------------
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
def _content_sections(nonlatin_hits, latin_foreign, signature_hits) -> int:
|
|
1396
|
+
"""Print the content sections; return 1 if any unexpected hit or signature."""
|
|
1397
|
+
|
|
1398
|
+
def show(label: str, hits: list[tuple], color: str) -> int:
|
|
1399
|
+
unexpected = [h for h in hits if not h[3]]
|
|
1400
|
+
expected = [h for h in hits if h[3]]
|
|
1401
|
+
if hits:
|
|
1402
|
+
print(f"{color}{BOLD}{label} ({len(hits)}){RESET}")
|
|
1403
|
+
for book_id, title, tag, _exp, detail in sorted(unexpected):
|
|
1404
|
+
print(f" {RED}#{book_id}{RESET} [{tag}] {title}")
|
|
1405
|
+
print(f" {detail}")
|
|
1406
|
+
for book_id, title, tag, _exp, detail in sorted(expected):
|
|
1407
|
+
print(f" #{book_id} [{tag}] {title} {GREEN}(expected-foreign){RESET}")
|
|
1408
|
+
print(f" {detail}")
|
|
1409
|
+
|
|
1410
|
+
return len(unexpected)
|
|
1411
|
+
|
|
1412
|
+
unexpected = 0
|
|
1413
|
+
unexpected += show("NON-LATIN SCRIPT", nonlatin_hits, RED)
|
|
1414
|
+
unexpected += show("LATIN-SCRIPT FOREIGN (stopword vote)", latin_foreign, RED)
|
|
1415
|
+
# An injected piracy/ad notice is a defect no matter what language the book
|
|
1416
|
+
# is declared in, so the expected-foreign flag never applies here: without
|
|
1417
|
+
# this, a signature hit on a declared-foreign book printed "(expected-
|
|
1418
|
+
# foreign)" and "0 file(s) need review" while still failing the run.
|
|
1419
|
+
signature_hits = [(b, t, g, False, d) for b, t, g, _e, d in signature_hits]
|
|
1420
|
+
unexpected += show("INJECTION SIGNATURE", signature_hits, YELLOW)
|
|
1421
|
+
if unexpected == 0 and not signature_hits:
|
|
1422
|
+
print(f"{GREEN}{BOLD}content CLEAN{RESET}: no unexpected foreign content.")
|
|
1423
|
+
return 0
|
|
1424
|
+
print(
|
|
1425
|
+
f"{RED}{BOLD}content FOUND{RESET}: {unexpected} file(s) need review "
|
|
1426
|
+
f"(replace wrong-language editions with English copies)."
|
|
1427
|
+
)
|
|
1428
|
+
return 1
|
|
1429
|
+
|
|
1430
|
+
|
|
1431
|
+
def _pagenum_sections(found) -> int:
|
|
1432
|
+
if found:
|
|
1433
|
+
print(f"{RED}{BOLD}BAKED-IN PAGE NUMBERS ({len(found)}){RESET}")
|
|
1434
|
+
for book_id, title, tag, r in sorted(found, key=lambda x: -x[3]["n_hits"]):
|
|
1435
|
+
mark = f" {YELLOW}[watermark]{RESET}" if r["watermark"] else ""
|
|
1436
|
+
print(
|
|
1437
|
+
f" {RED}#{book_id}{RESET} [{tag}] {title}{mark}\n"
|
|
1438
|
+
f" {r['n_hits']} baked numbers, {r['span'] * 100:.0f}% of book, run {r['run']}"
|
|
1439
|
+
)
|
|
1440
|
+
for ex in r["examples"][:3]:
|
|
1441
|
+
print(f" ...{ex['prev']} {BOLD}{ex['v']}{RESET} {ex['next']}...")
|
|
1442
|
+
print()
|
|
1443
|
+
print(
|
|
1444
|
+
f"{RED}{BOLD}pagenumbers FOUND{RESET}: {len(found)} file(s) need review "
|
|
1445
|
+
f"(re-source and replace bad conversions)."
|
|
1446
|
+
)
|
|
1447
|
+
return 1
|
|
1448
|
+
print(f"{GREEN}{BOLD}pagenumbers CLEAN{RESET}: no baked-in page numbers found.")
|
|
1449
|
+
return 0
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _empty_sections(empty, partial, thin) -> int:
|
|
1453
|
+
if empty:
|
|
1454
|
+
print(f"{RED}{BOLD}EMPTY / NO TEXT ({len(empty)}){RESET}")
|
|
1455
|
+
for book_id, title, tag, r in sorted(empty, key=lambda x: x[3]["chars"]):
|
|
1456
|
+
print(f" {RED}#{book_id}{RESET} [{tag}] {title}\n {_empty_detail(r)}")
|
|
1457
|
+
print()
|
|
1458
|
+
if partial:
|
|
1459
|
+
print(f"{RED}{BOLD}PARTIAL / PLACEHOLDER EXPORT ({len(partial)}){RESET}")
|
|
1460
|
+
for book_id, title, tag, r in sorted(partial, key=lambda x: x[3]["chars"]):
|
|
1461
|
+
print(f" {RED}#{book_id}{RESET} [{tag}] {title}\n {_empty_detail(r)}")
|
|
1462
|
+
print()
|
|
1463
|
+
if thin:
|
|
1464
|
+
print(
|
|
1465
|
+
f"{YELLOW}{BOLD}THIN (review; may be legitimately short) ({len(thin)}){RESET}"
|
|
1466
|
+
)
|
|
1467
|
+
for book_id, title, tag, r in sorted(thin, key=lambda x: x[3]["chars"]):
|
|
1468
|
+
print(
|
|
1469
|
+
f" {YELLOW}#{book_id}{RESET} [{tag}] {title}\n {_empty_detail(r)}"
|
|
1470
|
+
)
|
|
1471
|
+
print()
|
|
1472
|
+
found = len(empty) + len(partial)
|
|
1473
|
+
if found:
|
|
1474
|
+
print(f"{RED}{BOLD}emptytext FOUND{RESET}: {found} file(s) need re-sourcing.")
|
|
1475
|
+
return 1
|
|
1476
|
+
suffix = f" ({len(thin)} thin, advisory)" if thin else ""
|
|
1477
|
+
print(f"{GREEN}{BOLD}emptytext CLEAN{RESET}: every EPUB has body text{suffix}.")
|
|
1478
|
+
return 0
|
|
1479
|
+
|
|
1480
|
+
|
|
1481
|
+
def _ocr_sections(found) -> int:
|
|
1482
|
+
if found:
|
|
1483
|
+
print(f"{RED}{BOLD}OCR-DAMAGED PROSE ({len(found)}){RESET}")
|
|
1484
|
+
for book_id, title, tag, r in sorted(found, key=lambda x: -x[3]["split_rate"]):
|
|
1485
|
+
print(f" {RED}#{book_id}{RESET} [{tag}] {title}\n {_ocr_detail(r)}")
|
|
1486
|
+
for ex in r["examples"][:3]:
|
|
1487
|
+
print(f" ...{ex['prev']} {BOLD}/{RESET} {ex['next']}...")
|
|
1488
|
+
print()
|
|
1489
|
+
print(
|
|
1490
|
+
f"{RED}{BOLD}ocr FOUND{RESET}: {len(found)} file(s) need review "
|
|
1491
|
+
f"(re-source and replace damaged conversions)."
|
|
1492
|
+
)
|
|
1493
|
+
return 1
|
|
1494
|
+
print(f"{GREEN}{BOLD}ocr CLEAN{RESET}: no OCR-damaged prose found.")
|
|
1495
|
+
return 0
|
|
1496
|
+
|
|
1497
|
+
|
|
1498
|
+
def _monolithic_sections(hits) -> int:
|
|
1499
|
+
if hits:
|
|
1500
|
+
print(f"{RED}{BOLD}MONOLITHIC DOCUMENTS ({len(hits)}){RESET}")
|
|
1501
|
+
for book_id, title, tag, r in sorted(
|
|
1502
|
+
hits, key=lambda x: -x[3]["max_doc_chars"]
|
|
1503
|
+
):
|
|
1504
|
+
print(f" {RED}#{book_id}{RESET} [{tag}] {title}")
|
|
1505
|
+
print(
|
|
1506
|
+
f" max doc {r['max_doc_chars']:,} chars ({r['worst_doc']});"
|
|
1507
|
+
f" spine {r['spine_len']}"
|
|
1508
|
+
)
|
|
1509
|
+
print()
|
|
1510
|
+
print(
|
|
1511
|
+
f"{RED}{BOLD}monolithic FOUND{RESET}: {len(hits)} file(s) need review "
|
|
1512
|
+
f"(readers may refuse to render; re-source or split by hand)."
|
|
1513
|
+
)
|
|
1514
|
+
return 1
|
|
1515
|
+
print(f"{GREEN}{BOLD}monolithic CLEAN{RESET}: no oversized single document.")
|
|
1516
|
+
return 0
|
|
1517
|
+
|
|
1518
|
+
|
|
1519
|
+
def _corrupt_sections(hits) -> int:
|
|
1520
|
+
if hits:
|
|
1521
|
+
print(f"{RED}{BOLD}CORRUPT ARCHIVES ({len(hits)}){RESET}")
|
|
1522
|
+
for book_id, title, tag, n, first in sorted(hits):
|
|
1523
|
+
print(f" {RED}#{book_id}{RESET} [{tag}] {title}")
|
|
1524
|
+
print(f" corrupt:{n} (first: {first}) — damaged archive; re-source")
|
|
1525
|
+
print()
|
|
1526
|
+
print(
|
|
1527
|
+
f"{RED}{BOLD}archive FOUND{RESET}: {len(hits)} file(s) need re-sourcing "
|
|
1528
|
+
f"(a damaged archive is not a content decision)."
|
|
1529
|
+
)
|
|
1530
|
+
return 1
|
|
1531
|
+
print(f"{GREEN}{BOLD}archive CLEAN{RESET}: every entry fully readable.")
|
|
1532
|
+
return 0
|
|
1533
|
+
|
|
1534
|
+
|
|
1535
|
+
def _spine_sections(hits, advisory) -> int:
|
|
1536
|
+
if advisory:
|
|
1537
|
+
print(
|
|
1538
|
+
f"{YELLOW}{BOLD}TOC MANIFEST CONVENTION ({len(advisory)}; reported, not flagged){RESET}"
|
|
1539
|
+
)
|
|
1540
|
+
for book_id, title, tag, r in sorted(advisory):
|
|
1541
|
+
print(f" {YELLOW}#{book_id}{RESET} [{tag}] {title}")
|
|
1542
|
+
print(f" toc {r['refs']} refs, {r['absent']} absent ({r['class']})")
|
|
1543
|
+
print()
|
|
1544
|
+
if hits:
|
|
1545
|
+
print(f"{RED}{BOLD}SPINE FRAGMENTS ({len(hits)}){RESET}")
|
|
1546
|
+
for book_id, title, tag, r in sorted(hits):
|
|
1547
|
+
print(f" {RED}#{book_id}{RESET} [{tag}] {title}")
|
|
1548
|
+
print(f" toc {r['refs']} refs, {r['absent']} absent — span broken")
|
|
1549
|
+
print()
|
|
1550
|
+
print(
|
|
1551
|
+
f"{RED}{BOLD}spine FOUND{RESET}: {len(hits)} file(s) are fragments "
|
|
1552
|
+
f"of themselves; quarantine."
|
|
1553
|
+
)
|
|
1554
|
+
return 1
|
|
1555
|
+
if not advisory:
|
|
1556
|
+
print(f"{GREEN}{BOLD}spine CLEAN{RESET}: every ToC target present.")
|
|
1557
|
+
return 0
|
|
1558
|
+
|
|
1559
|
+
|
|
1560
|
+
# ----------------------------------------------------------------------------
|
|
1561
|
+
# Runner
|
|
1562
|
+
# ----------------------------------------------------------------------------
|
|
1563
|
+
|
|
1564
|
+
ALL: tuple[str, ...] = ("content", "pagenumbers", "emptytext", "ocr", "monolithic")
|
|
1565
|
+
|
|
1566
|
+
|
|
1567
|
+
def run_library(
|
|
1568
|
+
selected: list[str],
|
|
1569
|
+
min_chars: int,
|
|
1570
|
+
thin_chars: int,
|
|
1571
|
+
tag: str | None = None,
|
|
1572
|
+
max_doc_chars: int = DEFAULT_MAX_DOC_CHARS,
|
|
1573
|
+
) -> int:
|
|
1574
|
+
# The scan loop below reuses `tag` for its per-book display column; keep
|
|
1575
|
+
# the --tag argument under a distinct name so it survives that shadowing.
|
|
1576
|
+
audit_tag = tag
|
|
1577
|
+
library_root = resolve_library_root()
|
|
1578
|
+
if library_root is None:
|
|
1579
|
+
print(
|
|
1580
|
+
"ERROR: no metadata.db next to this script or in the current "
|
|
1581
|
+
"directory. Run from the library directory."
|
|
1582
|
+
)
|
|
1583
|
+
return 2
|
|
1584
|
+
db_path = library_root / "metadata.db"
|
|
1585
|
+
|
|
1586
|
+
from cquarry.db import CalibreDB
|
|
1587
|
+
|
|
1588
|
+
try:
|
|
1589
|
+
db = CalibreDB(str(db_path))
|
|
1590
|
+
except Exception as e:
|
|
1591
|
+
print(f"ERROR: cannot open {db_path}: {e}")
|
|
1592
|
+
return 2
|
|
1593
|
+
try:
|
|
1594
|
+
cur = db.conn.cursor()
|
|
1595
|
+
booktags: dict[int, list[str]] = {}
|
|
1596
|
+
for bid, tname in cur.execute(
|
|
1597
|
+
"SELECT bt.book, t.name FROM books_tags_link bt JOIN tags t ON t.id = bt.tag"
|
|
1598
|
+
):
|
|
1599
|
+
booktags.setdefault(bid, []).append(tname)
|
|
1600
|
+
declared: dict[int, set[str]] = {}
|
|
1601
|
+
for bid, lang in cur.execute(
|
|
1602
|
+
"SELECT bl.book, l.lang_code FROM books_languages_link bl "
|
|
1603
|
+
"JOIN languages l ON l.id = bl.lang_code"
|
|
1604
|
+
):
|
|
1605
|
+
declared.setdefault(bid, set()).add(lang)
|
|
1606
|
+
rows = cur.execute(
|
|
1607
|
+
"SELECT b.id, b.title FROM books b "
|
|
1608
|
+
"JOIN data d ON d.book = b.id WHERE d.format = 'EPUB' ORDER BY b.id"
|
|
1609
|
+
).fetchall()
|
|
1610
|
+
# Canonical path resolution (library root / books.path / name.epub)
|
|
1611
|
+
# comes from cquarry so the layout logic lives in exactly one place.
|
|
1612
|
+
epubs = [
|
|
1613
|
+
(
|
|
1614
|
+
bid,
|
|
1615
|
+
title,
|
|
1616
|
+
Path(db.get_format_path(bid, "EPUB", verify=False)),
|
|
1617
|
+
)
|
|
1618
|
+
for bid, title in rows
|
|
1619
|
+
]
|
|
1620
|
+
finally:
|
|
1621
|
+
db.close()
|
|
1622
|
+
|
|
1623
|
+
nonlatin_hits: list[tuple] = []
|
|
1624
|
+
latin_foreign: list[tuple] = []
|
|
1625
|
+
signature_hits: list[tuple] = []
|
|
1626
|
+
pagenum_found: list[tuple] = []
|
|
1627
|
+
empty_hits: list[tuple] = []
|
|
1628
|
+
partial_hits: list[tuple] = []
|
|
1629
|
+
thin_hits: list[tuple] = []
|
|
1630
|
+
ocr_found: list[tuple] = []
|
|
1631
|
+
mono_hits: list[tuple] = []
|
|
1632
|
+
corrupt_hits: list[tuple] = []
|
|
1633
|
+
spine_hits: list[tuple] = []
|
|
1634
|
+
spine_advisory: list[tuple] = []
|
|
1635
|
+
errors: list[tuple] = []
|
|
1636
|
+
scanned = 0
|
|
1637
|
+
|
|
1638
|
+
for book_id, title, full in ui.tqdm(epubs, desc=ui.info("Scanning library")):
|
|
1639
|
+
tags = booktags.get(book_id, [])
|
|
1640
|
+
tag = tags[0] if tags else "?"
|
|
1641
|
+
try:
|
|
1642
|
+
book = load_book(full)
|
|
1643
|
+
except Exception as e:
|
|
1644
|
+
errors.append((book_id, title, tag, f"{type(e).__name__}: {e}"))
|
|
1645
|
+
continue
|
|
1646
|
+
scanned += 1
|
|
1647
|
+
corrupt_r = analyze_corrupt(book)
|
|
1648
|
+
if corrupt_r["n"]:
|
|
1649
|
+
corrupt_hits.append(
|
|
1650
|
+
(book_id, title, tag, corrupt_r["n"], corrupt_r["first"])
|
|
1651
|
+
)
|
|
1652
|
+
spine_r = spine_integrity(book)
|
|
1653
|
+
if spine_r["class"] == "fragment":
|
|
1654
|
+
spine_hits.append((book_id, title, tag, spine_r))
|
|
1655
|
+
elif spine_r["class"] != "ok":
|
|
1656
|
+
spine_advisory.append((book_id, title, tag, spine_r))
|
|
1657
|
+
|
|
1658
|
+
if "content" in selected:
|
|
1659
|
+
r = analyze_content(book)
|
|
1660
|
+
langs = declared.get(book_id, set())
|
|
1661
|
+
decl = ",".join(sorted(langs)) if langs else "?"
|
|
1662
|
+
expected = any(lang != "eng" for lang in langs) or any(
|
|
1663
|
+
t.startswith("NonFic.Language.") for t in tags
|
|
1664
|
+
)
|
|
1665
|
+
for category, detail in findings(r):
|
|
1666
|
+
if category == "NON-LATIN SCRIPT":
|
|
1667
|
+
nonlatin_hits.append(
|
|
1668
|
+
(book_id, title, tag, expected, f"{detail}; declared={decl}")
|
|
1669
|
+
)
|
|
1670
|
+
elif category == "LATIN-SCRIPT FOREIGN":
|
|
1671
|
+
latin_foreign.append(
|
|
1672
|
+
(book_id, title, tag, expected, f"{detail}; declared={decl}")
|
|
1673
|
+
)
|
|
1674
|
+
else:
|
|
1675
|
+
signature_hits.append((book_id, title, tag, expected, detail))
|
|
1676
|
+
|
|
1677
|
+
if "pagenumbers" in selected:
|
|
1678
|
+
r = analyze_pagenumbers(book)
|
|
1679
|
+
if is_defective(r):
|
|
1680
|
+
pagenum_found.append((book_id, title, tag, r))
|
|
1681
|
+
|
|
1682
|
+
if "emptytext" in selected and not corrupt_r["n"]:
|
|
1683
|
+
r = analyze_emptytext(book)
|
|
1684
|
+
verdict = classify(r, min_chars, thin_chars)
|
|
1685
|
+
if verdict == "EMPTY":
|
|
1686
|
+
empty_hits.append((book_id, title, tag, r))
|
|
1687
|
+
elif verdict == "PARTIAL":
|
|
1688
|
+
partial_hits.append((book_id, title, tag, r))
|
|
1689
|
+
elif verdict == "THIN":
|
|
1690
|
+
thin_hits.append((book_id, title, tag, r))
|
|
1691
|
+
|
|
1692
|
+
if "ocr" in selected:
|
|
1693
|
+
r = analyze_ocr(book)
|
|
1694
|
+
if is_ocr_damaged(r):
|
|
1695
|
+
ocr_found.append((book_id, title, tag, r))
|
|
1696
|
+
|
|
1697
|
+
if "monolithic" in selected:
|
|
1698
|
+
r = analyze_monolithic(book)
|
|
1699
|
+
if is_monolithic(r, max_doc_chars):
|
|
1700
|
+
mono_hits.append((book_id, title, tag, r))
|
|
1701
|
+
|
|
1702
|
+
print(f"Scanned {scanned} EPUBs in {library_root}\n")
|
|
1703
|
+
rc = 0
|
|
1704
|
+
multi = len(selected) > 1
|
|
1705
|
+
for key in ALL:
|
|
1706
|
+
if key not in selected:
|
|
1707
|
+
continue
|
|
1708
|
+
if multi:
|
|
1709
|
+
print(f"{BOLD}== {key} =={RESET}")
|
|
1710
|
+
if key == "content":
|
|
1711
|
+
rc |= _content_sections(nonlatin_hits, latin_foreign, signature_hits)
|
|
1712
|
+
elif key == "pagenumbers":
|
|
1713
|
+
rc |= _pagenum_sections(pagenum_found)
|
|
1714
|
+
elif key == "emptytext":
|
|
1715
|
+
rc |= _empty_sections(empty_hits, partial_hits, thin_hits)
|
|
1716
|
+
elif key == "archive":
|
|
1717
|
+
rc |= _corrupt_sections(corrupt_hits)
|
|
1718
|
+
elif key == "spine":
|
|
1719
|
+
rc |= _spine_sections(spine_hits, spine_advisory)
|
|
1720
|
+
elif key == "monolithic":
|
|
1721
|
+
rc |= _monolithic_sections(mono_hits)
|
|
1722
|
+
else:
|
|
1723
|
+
rc |= _ocr_sections(ocr_found)
|
|
1724
|
+
if multi:
|
|
1725
|
+
print()
|
|
1726
|
+
|
|
1727
|
+
if errors:
|
|
1728
|
+
print(f"{YELLOW}{BOLD}SCAN ERRORS ({len(errors)}){RESET}")
|
|
1729
|
+
for book_id, title, tag, msg in errors:
|
|
1730
|
+
print(f" #{book_id} [{tag}] {title}\n {msg}")
|
|
1731
|
+
print()
|
|
1732
|
+
rc |= 1
|
|
1733
|
+
|
|
1734
|
+
if audit_tag:
|
|
1735
|
+
rc |= _apply_audit_tag(
|
|
1736
|
+
library_root,
|
|
1737
|
+
audit_tag,
|
|
1738
|
+
{
|
|
1739
|
+
"content": [h[0] for h in nonlatin_hits]
|
|
1740
|
+
+ [h[0] for h in latin_foreign]
|
|
1741
|
+
+ [h[0] for h in signature_hits],
|
|
1742
|
+
"pagenumbers": [h[0] for h in pagenum_found],
|
|
1743
|
+
"emptytext": [
|
|
1744
|
+
h[0] for h in empty_hits + partial_hits
|
|
1745
|
+
], # THIN is advisory and stays untagged
|
|
1746
|
+
"ocr": [h[0] for h in ocr_found],
|
|
1747
|
+
"monolithic": [h[0] for h in mono_hits],
|
|
1748
|
+
"archive": [h[0] for h in corrupt_hits],
|
|
1749
|
+
"spine": [h[0] for h in spine_hits],
|
|
1750
|
+
},
|
|
1751
|
+
)
|
|
1752
|
+
return rc
|
|
1753
|
+
|
|
1754
|
+
|
|
1755
|
+
def _apply_audit_tag(
|
|
1756
|
+
library_root: Path, tag: str, flagged: dict[str, list[int]]
|
|
1757
|
+
) -> int:
|
|
1758
|
+
"""Apply ``tag`` to every flagged book via cquarry's opt-in write path.
|
|
1759
|
+
|
|
1760
|
+
Only reached with the explicit --tag flag; the audit itself stays strictly
|
|
1761
|
+
read-only. Calibre should be closed so the write does not fight its lock.
|
|
1762
|
+
Returns 0 on success, 2 on setup failure.
|
|
1763
|
+
"""
|
|
1764
|
+
from cquarry.write import WritableCalibreDB
|
|
1765
|
+
|
|
1766
|
+
ids = sorted({bid for bids in flagged.values() for bid in bids})
|
|
1767
|
+
if not ids:
|
|
1768
|
+
print("Nothing flagged; no tags applied.")
|
|
1769
|
+
return 0
|
|
1770
|
+
db_path = library_root / "metadata.db"
|
|
1771
|
+
try:
|
|
1772
|
+
with WritableCalibreDB(str(db_path)) as wdb:
|
|
1773
|
+
applied = 0
|
|
1774
|
+
for bid in ids:
|
|
1775
|
+
if wdb.add_tag(bid, tag):
|
|
1776
|
+
applied += 1
|
|
1777
|
+
except Exception as e:
|
|
1778
|
+
print(f"ERROR: tagging failed ({type(e).__name__}: {e}).")
|
|
1779
|
+
print("Is Calibre closed? The write path refuses to fight its lock.")
|
|
1780
|
+
return 2
|
|
1781
|
+
print(
|
|
1782
|
+
f"Tagged {applied} of {len(ids)} flagged books with [{tag}]"
|
|
1783
|
+
" (already-tagged books skipped)."
|
|
1784
|
+
)
|
|
1785
|
+
return 0
|
|
1786
|
+
|
|
1787
|
+
|
|
1788
|
+
def _content_dir(r: dict) -> tuple[bool, str, list[str]]:
|
|
1789
|
+
hits = findings(r)
|
|
1790
|
+
if not hits:
|
|
1791
|
+
return False, "OK", []
|
|
1792
|
+
return True, "REVIEW", [f"{cat}: {detail}" for cat, detail in hits]
|
|
1793
|
+
|
|
1794
|
+
|
|
1795
|
+
def _pagenum_dir(r: dict) -> tuple[bool, str, list[str]]:
|
|
1796
|
+
if not is_defective(r):
|
|
1797
|
+
return False, "OK", []
|
|
1798
|
+
mark = " [watermark]" if r["watermark"] else ""
|
|
1799
|
+
lines = [
|
|
1800
|
+
f"{r['n_hits']} baked numbers, {r['span'] * 100:.0f}% of book, run {r['run']}{mark}"
|
|
1801
|
+
]
|
|
1802
|
+
lines += [
|
|
1803
|
+
f"...{ex['prev']} {ex['v']} {ex['next']}..." for ex in r["examples"][:2]
|
|
1804
|
+
]
|
|
1805
|
+
return True, "REVIEW", lines
|
|
1806
|
+
|
|
1807
|
+
|
|
1808
|
+
def _empty_dir(r: dict, min_chars: int, thin_chars: int) -> tuple[bool, str, list[str]]:
|
|
1809
|
+
verdict = classify(r, min_chars, thin_chars)
|
|
1810
|
+
if verdict == "OK":
|
|
1811
|
+
return False, "OK", []
|
|
1812
|
+
if verdict == "CORRUPT":
|
|
1813
|
+
return True, "CORRUPT", [_empty_detail(r)]
|
|
1814
|
+
return verdict in ("EMPTY", "PARTIAL"), verdict, [_empty_detail(r)]
|
|
1815
|
+
|
|
1816
|
+
|
|
1817
|
+
def _monolithic_dir(r: dict, max_doc_chars: int) -> tuple[bool, str, list[str]]:
|
|
1818
|
+
if is_monolithic(r, max_doc_chars):
|
|
1819
|
+
return (
|
|
1820
|
+
True,
|
|
1821
|
+
"FLAG",
|
|
1822
|
+
[f"max doc {r['max_doc_chars']:,} chars ({r['worst_doc']})"],
|
|
1823
|
+
)
|
|
1824
|
+
return False, "OK", [f"max doc {r['max_doc_chars']:,} chars"]
|
|
1825
|
+
|
|
1826
|
+
|
|
1827
|
+
def _ocr_dir(r: dict) -> tuple[bool, str, list[str]]:
|
|
1828
|
+
if not is_ocr_damaged(r):
|
|
1829
|
+
return False, "OK", []
|
|
1830
|
+
lines = [_ocr_detail(r)]
|
|
1831
|
+
lines += [f"...{ex['prev']} / {ex['next']}..." for ex in r["examples"][:2]]
|
|
1832
|
+
return True, "REVIEW", lines
|
|
1833
|
+
|
|
1834
|
+
|
|
1835
|
+
def run_directory(
|
|
1836
|
+
directory: Path,
|
|
1837
|
+
selected: list[str],
|
|
1838
|
+
min_chars: int,
|
|
1839
|
+
thin_chars: int,
|
|
1840
|
+
max_doc_chars: int = DEFAULT_MAX_DOC_CHARS,
|
|
1841
|
+
) -> int:
|
|
1842
|
+
if not directory.is_dir():
|
|
1843
|
+
print(f"ERROR: {directory} is not a directory.")
|
|
1844
|
+
return 2
|
|
1845
|
+
epubs = sorted(directory.rglob("*.epub"))
|
|
1846
|
+
if not epubs:
|
|
1847
|
+
print(f"No .epub files found under {directory}")
|
|
1848
|
+
return 2
|
|
1849
|
+
|
|
1850
|
+
print(f"Auditing {len(epubs)} EPUB(s) in {directory}\n")
|
|
1851
|
+
multi = len(selected) > 1
|
|
1852
|
+
problems = 0
|
|
1853
|
+
errors = 0
|
|
1854
|
+
for path in ui.tqdm(epubs, desc=ui.info("Scanning directory")):
|
|
1855
|
+
try:
|
|
1856
|
+
book = load_book(path)
|
|
1857
|
+
except Exception as e:
|
|
1858
|
+
ui.tqdm.write(
|
|
1859
|
+
f" {YELLOW}ERROR {RESET} {path.name}\n {type(e).__name__}: {e}"
|
|
1860
|
+
)
|
|
1861
|
+
errors += 1
|
|
1862
|
+
continue
|
|
1863
|
+
|
|
1864
|
+
verdicts = []
|
|
1865
|
+
corrupt_r = analyze_corrupt(book)
|
|
1866
|
+
spine_r = spine_integrity(book)
|
|
1867
|
+
for key in ALL:
|
|
1868
|
+
if key not in selected:
|
|
1869
|
+
continue
|
|
1870
|
+
if key == "emptytext" and corrupt_r["n"]:
|
|
1871
|
+
continue # the archive verdict owns this book's body-text story
|
|
1872
|
+
if key == "content":
|
|
1873
|
+
problem, status, lines = _content_dir(analyze_content(book))
|
|
1874
|
+
elif key == "pagenumbers":
|
|
1875
|
+
problem, status, lines = _pagenum_dir(analyze_pagenumbers(book))
|
|
1876
|
+
elif key == "emptytext":
|
|
1877
|
+
problem, status, lines = _empty_dir(
|
|
1878
|
+
analyze_emptytext(book), min_chars, thin_chars
|
|
1879
|
+
)
|
|
1880
|
+
elif key == "monolithic":
|
|
1881
|
+
problem, status, lines = _monolithic_dir(
|
|
1882
|
+
analyze_monolithic(book), max_doc_chars
|
|
1883
|
+
)
|
|
1884
|
+
else:
|
|
1885
|
+
problem, status, lines = _ocr_dir(analyze_ocr(book))
|
|
1886
|
+
if problem:
|
|
1887
|
+
problems += 1
|
|
1888
|
+
verdicts.append((key, problem, status, lines))
|
|
1889
|
+
|
|
1890
|
+
if corrupt_r["n"]:
|
|
1891
|
+
problem, status, lines = _corrupt_verdict(corrupt_r)
|
|
1892
|
+
problems += 1
|
|
1893
|
+
verdicts.append(("archive", problem, status, lines))
|
|
1894
|
+
|
|
1895
|
+
if spine_r["class"] != "ok":
|
|
1896
|
+
problem, status, lines = _spine_verdict(spine_r)
|
|
1897
|
+
if problem:
|
|
1898
|
+
problems += 1
|
|
1899
|
+
verdicts.append(("spine", problem, status, lines))
|
|
1900
|
+
|
|
1901
|
+
if multi:
|
|
1902
|
+
ui.tqdm.write(f" {path.name}")
|
|
1903
|
+
for key, problem, status, lines in verdicts:
|
|
1904
|
+
color = RED if problem else (YELLOW if status != "OK" else GREEN)
|
|
1905
|
+
ui.tqdm.write(f" {color}{status:<6}{RESET} {key}")
|
|
1906
|
+
for ln in lines:
|
|
1907
|
+
ui.tqdm.write(f" {ln}")
|
|
1908
|
+
else:
|
|
1909
|
+
_key, problem, status, lines = verdicts[0]
|
|
1910
|
+
color = RED if problem else (YELLOW if status != "OK" else GREEN)
|
|
1911
|
+
ui.tqdm.write(f" {color}{status:<6}{RESET} {path.name}")
|
|
1912
|
+
for ln in lines:
|
|
1913
|
+
ui.tqdm.write(f" {ln}")
|
|
1914
|
+
print()
|
|
1915
|
+
|
|
1916
|
+
if problems == 0 and errors == 0:
|
|
1917
|
+
print(f"{GREEN}{BOLD}CLEAN{RESET}: no problems in {len(epubs)} file(s).")
|
|
1918
|
+
return 0
|
|
1919
|
+
print(
|
|
1920
|
+
f"{RED}{BOLD}FOUND{RESET}: {problems} problem(s) need review, "
|
|
1921
|
+
f"{errors} scan error(s)."
|
|
1922
|
+
)
|
|
1923
|
+
return 1
|
|
1924
|
+
|
|
1925
|
+
|
|
1926
|
+
def run_single(
|
|
1927
|
+
book_id: int,
|
|
1928
|
+
selected: list[str],
|
|
1929
|
+
min_chars: int,
|
|
1930
|
+
thin_chars: int,
|
|
1931
|
+
tag: str | None = None,
|
|
1932
|
+
max_doc_chars: int = DEFAULT_MAX_DOC_CHARS,
|
|
1933
|
+
) -> int:
|
|
1934
|
+
"""Audit one library book by id — cquarry's single-entity fetch.
|
|
1935
|
+
|
|
1936
|
+
Uses :meth:`CalibreDB.get_book` so only that book's row is read instead
|
|
1937
|
+
of caching the entire library layout, then resolves the EPUB through
|
|
1938
|
+
cquarry's ``get_format_path``. The audit itself stays strictly read-only;
|
|
1939
|
+
``--tag`` applies via cquarry's opt-in write path only when flagged.
|
|
1940
|
+
"""
|
|
1941
|
+
library_root = resolve_library_root()
|
|
1942
|
+
if library_root is None:
|
|
1943
|
+
print(
|
|
1944
|
+
"ERROR: no metadata.db next to this script or in the current "
|
|
1945
|
+
"directory. Run from the library directory."
|
|
1946
|
+
)
|
|
1947
|
+
return 2
|
|
1948
|
+
|
|
1949
|
+
from cquarry.db import CalibreDB
|
|
1950
|
+
|
|
1951
|
+
try:
|
|
1952
|
+
db = CalibreDB(str(library_root / "metadata.db"))
|
|
1953
|
+
except Exception as e:
|
|
1954
|
+
print(f"ERROR: cannot open {library_root / 'metadata.db'}: {e}")
|
|
1955
|
+
return 2
|
|
1956
|
+
try:
|
|
1957
|
+
rec = db.get_book(book_id)
|
|
1958
|
+
if rec is None:
|
|
1959
|
+
print(f"ERROR: no book #{book_id} in {library_root}")
|
|
1960
|
+
return 2
|
|
1961
|
+
title = rec["title"]
|
|
1962
|
+
tags = rec["tags"]
|
|
1963
|
+
try:
|
|
1964
|
+
path = Path(db.get_format_path(book_id, "EPUB", verify=True))
|
|
1965
|
+
except (ValueError, FileNotFoundError) as e:
|
|
1966
|
+
print(f"ERROR: book #{book_id} ({title}): {e}")
|
|
1967
|
+
return 2
|
|
1968
|
+
finally:
|
|
1969
|
+
db.close()
|
|
1970
|
+
|
|
1971
|
+
try:
|
|
1972
|
+
book = load_book(path)
|
|
1973
|
+
except Exception as e:
|
|
1974
|
+
print(f"ERROR reading {path.name}: {type(e).__name__}: {e}")
|
|
1975
|
+
return 1
|
|
1976
|
+
|
|
1977
|
+
tag_display = tags[0] if tags else "?"
|
|
1978
|
+
print(f"Auditing #{book_id} [{tag_display}] {title}\n {path}\n")
|
|
1979
|
+
problems = 0
|
|
1980
|
+
multi = len(selected) > 1
|
|
1981
|
+
verdicts = []
|
|
1982
|
+
corrupt_r = analyze_corrupt(book)
|
|
1983
|
+
spine_r = spine_integrity(book)
|
|
1984
|
+
for key in ALL:
|
|
1985
|
+
if key not in selected:
|
|
1986
|
+
continue
|
|
1987
|
+
if key == "emptytext" and corrupt_r["n"]:
|
|
1988
|
+
# Corruption owns the body-text story for this book; reporting
|
|
1989
|
+
# EMPTY here would be the wrong disease.
|
|
1990
|
+
continue
|
|
1991
|
+
if key == "content":
|
|
1992
|
+
problem, status, lines = _content_dir(analyze_content(book))
|
|
1993
|
+
elif key == "pagenumbers":
|
|
1994
|
+
problem, status, lines = _pagenum_dir(analyze_pagenumbers(book))
|
|
1995
|
+
elif key == "emptytext":
|
|
1996
|
+
problem, status, lines = _empty_dir(
|
|
1997
|
+
analyze_emptytext(book), min_chars, thin_chars
|
|
1998
|
+
)
|
|
1999
|
+
elif key == "monolithic":
|
|
2000
|
+
problem, status, lines = _monolithic_dir(
|
|
2001
|
+
analyze_monolithic(book), max_doc_chars
|
|
2002
|
+
)
|
|
2003
|
+
else:
|
|
2004
|
+
problem, status, lines = _ocr_dir(analyze_ocr(book))
|
|
2005
|
+
if problem:
|
|
2006
|
+
problems += 1
|
|
2007
|
+
verdicts.append((key, problem, status, lines))
|
|
2008
|
+
|
|
2009
|
+
if corrupt_r["n"]:
|
|
2010
|
+
problem, status, lines = _corrupt_verdict(corrupt_r)
|
|
2011
|
+
problems += 1
|
|
2012
|
+
verdicts.append(("archive", problem, status, lines))
|
|
2013
|
+
|
|
2014
|
+
if spine_r["class"] != "ok":
|
|
2015
|
+
problem, status, lines = _spine_verdict(spine_r)
|
|
2016
|
+
if problem:
|
|
2017
|
+
problems += 1
|
|
2018
|
+
verdicts.append(("spine", problem, status, lines))
|
|
2019
|
+
|
|
2020
|
+
for key, problem, status, lines in verdicts:
|
|
2021
|
+
if multi:
|
|
2022
|
+
print(f" {key}")
|
|
2023
|
+
color = RED if problem else (YELLOW if status != "OK" else GREEN)
|
|
2024
|
+
prefix = " " if multi else " "
|
|
2025
|
+
print(f"{prefix}{color}{status:<6}{RESET} {key}")
|
|
2026
|
+
for ln in lines:
|
|
2027
|
+
print(f"{prefix} {ln}")
|
|
2028
|
+
|
|
2029
|
+
rc = 0
|
|
2030
|
+
if problems:
|
|
2031
|
+
rc = 1
|
|
2032
|
+
print(f"\n{RED}{BOLD}FOUND{RESET}: {problems} problem(s) need review.")
|
|
2033
|
+
else:
|
|
2034
|
+
print(f"\n{GREEN}{BOLD}CLEAN{RESET}: book #{book_id} passed the audit.")
|
|
2035
|
+
if tag and problems:
|
|
2036
|
+
rc |= _apply_audit_tag(
|
|
2037
|
+
library_root,
|
|
2038
|
+
tag,
|
|
2039
|
+
{key: [book_id] for key, problem, _s, _l in verdicts if problem},
|
|
2040
|
+
)
|
|
2041
|
+
return rc
|
|
2042
|
+
|
|
2043
|
+
|
|
2044
|
+
def main() -> int:
|
|
2045
|
+
parser = argparse.ArgumentParser(
|
|
2046
|
+
description="Audit EPUB body text for non-English content, baked-in page "
|
|
2047
|
+
"numbers, empty stubs, or OCR-damaged prose."
|
|
2048
|
+
)
|
|
2049
|
+
parser.add_argument(
|
|
2050
|
+
"mode",
|
|
2051
|
+
choices=("content", "pagenumbers", "emptytext", "ocr", "monolithic", "all"),
|
|
2052
|
+
help="which audit to run ('all' runs the analyzers in one decompression pass)",
|
|
2053
|
+
)
|
|
2054
|
+
parser.add_argument(
|
|
2055
|
+
"directory",
|
|
2056
|
+
nargs="?",
|
|
2057
|
+
help="vet loose .epub files under this directory instead of the library",
|
|
2058
|
+
)
|
|
2059
|
+
parser.add_argument(
|
|
2060
|
+
"--min-chars",
|
|
2061
|
+
type=int,
|
|
2062
|
+
default=DEFAULT_MIN_CHARS,
|
|
2063
|
+
help=f"emptytext EMPTY threshold (default {DEFAULT_MIN_CHARS})",
|
|
2064
|
+
)
|
|
2065
|
+
parser.add_argument(
|
|
2066
|
+
"--thin-chars",
|
|
2067
|
+
type=int,
|
|
2068
|
+
default=DEFAULT_THIN_CHARS,
|
|
2069
|
+
help=f"emptytext THIN advisory threshold (default {DEFAULT_THIN_CHARS})",
|
|
2070
|
+
)
|
|
2071
|
+
parser.add_argument(
|
|
2072
|
+
"--max-doc-chars",
|
|
2073
|
+
type=int,
|
|
2074
|
+
default=DEFAULT_MAX_DOC_CHARS,
|
|
2075
|
+
help=f"monolithic FLAG threshold (default {DEFAULT_MAX_DOC_CHARS})",
|
|
2076
|
+
)
|
|
2077
|
+
parser.add_argument(
|
|
2078
|
+
"--id",
|
|
2079
|
+
metavar="BOOK_IDS",
|
|
2080
|
+
default=None,
|
|
2081
|
+
help="audit library book(s) by Calibre id — one id or a comma-separated "
|
|
2082
|
+
"list (fetched via cquarry's single-entity get_book; cannot be "
|
|
2083
|
+
"combined with a directory)",
|
|
2084
|
+
)
|
|
2085
|
+
parser.add_argument(
|
|
2086
|
+
"--tag",
|
|
2087
|
+
metavar="TAG",
|
|
2088
|
+
default=None,
|
|
2089
|
+
help="library mode only: tag every flagged book via cquarry's opt-in "
|
|
2090
|
+
"write path (Calibre must be closed; audit itself stays read-only)",
|
|
2091
|
+
)
|
|
2092
|
+
args = parser.parse_args()
|
|
2093
|
+
ui.print_header("bindery audit - Execution")
|
|
2094
|
+
selected = list(ALL) if args.mode == "all" else [args.mode]
|
|
2095
|
+
if args.id is not None:
|
|
2096
|
+
if args.directory:
|
|
2097
|
+
print("ERROR: --id audits a library book; drop the directory argument.")
|
|
2098
|
+
return 2
|
|
2099
|
+
rc = 0
|
|
2100
|
+
for raw in str(args.id).split(","):
|
|
2101
|
+
raw = raw.strip()
|
|
2102
|
+
if not raw:
|
|
2103
|
+
continue
|
|
2104
|
+
try:
|
|
2105
|
+
bid = int(raw)
|
|
2106
|
+
except ValueError:
|
|
2107
|
+
print(f"ERROR: --id {raw!r} is not a book id.", file=sys.stderr)
|
|
2108
|
+
rc |= 2
|
|
2109
|
+
continue
|
|
2110
|
+
rc |= run_single(
|
|
2111
|
+
bid,
|
|
2112
|
+
selected,
|
|
2113
|
+
args.min_chars,
|
|
2114
|
+
args.thin_chars,
|
|
2115
|
+
tag=args.tag,
|
|
2116
|
+
max_doc_chars=args.max_doc_chars,
|
|
2117
|
+
)
|
|
2118
|
+
return rc
|
|
2119
|
+
if args.directory:
|
|
2120
|
+
return run_directory(
|
|
2121
|
+
Path(args.directory).expanduser(),
|
|
2122
|
+
selected,
|
|
2123
|
+
args.min_chars,
|
|
2124
|
+
args.thin_chars,
|
|
2125
|
+
max_doc_chars=args.max_doc_chars,
|
|
2126
|
+
)
|
|
2127
|
+
return run_library(
|
|
2128
|
+
selected,
|
|
2129
|
+
args.min_chars,
|
|
2130
|
+
args.thin_chars,
|
|
2131
|
+
tag=args.tag,
|
|
2132
|
+
max_doc_chars=args.max_doc_chars,
|
|
2133
|
+
)
|
|
2134
|
+
|
|
2135
|
+
|
|
2136
|
+
if __name__ == "__main__":
|
|
2137
|
+
sys.exit(main())
|