bindery-cli 0.27.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
bindery/audit.py ADDED
@@ -0,0 +1,2137 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ bindery audit: read the actual text of every EPUB and flag content problems that
4
+ metadata and structural validators cannot see. Four analyzers, one tool:
5
+
6
+ content non-English bodies (wrong-language editions) and injected
7
+ foreign-language ad-notices (declared lang=eng, body Portuguese
8
+ / Russian / etc.)
9
+ pagenumbers print page numbers (and running headers) captured as body *text*
10
+ by a bad PDF/OCR conversion, so they reflow into the middle of a
11
+ sentence ("where the hay cart 16 was taking him")
12
+ emptytext content-less stubs: a "Bookmate" export is cover/promo images
13
+ plus a tiny HTML placeholder, the spine pointing only at the
14
+ placeholder, the book itself absent (passes epubcheck and a
15
+ structural repairer because the one referenced doc is valid)
16
+ ocr OCR/conversion-damaged prose: paragraphs split mid-sentence
17
+ ("could just make out the shape" / "of another boat"), plus
18
+ dictionary-free side signals (en-dashes inside words, doubled
19
+ opening quotes, space-stripped proper nouns recurring alongside
20
+ their hyphenated form). Character-substitution errors ("sonic"
21
+ for "some") are OUT OF SCOPE: catching those needs a wordlist,
22
+ and this tool is stdlib-only by contract.
23
+ all run all four in a SINGLE decompression pass per book
24
+
25
+ This merges the former audit_epub_content.py / audit_epub_pagenumbers.py /
26
+ audit_epub_emptytext.py: they shared the same spine resolution, library/
27
+ directory dual-mode, read-only contract, and exit codes, and differed only in
28
+ the per-book verdict. `all` opens each EPUB once and feeds the decoded spine to
29
+ all four analyzers (the expensive part is decompression, so this is a real
30
+ win at library scale).
31
+
32
+ Companion to validate_metadata.py (which audits the catalogue) and to Bindery
33
+ (which repairs EPUB structure). This one reads body text and changes nothing;
34
+ it opens metadata.db strictly mode=ro.
35
+
36
+ Run from the library directory:
37
+ python3 bindery audit all # all four audits, whole library
38
+ python3 bindery audit content # one audit, whole library
39
+ python3 bindery audit all ~/Downloads # vet loose .epub files before import
40
+ python3 bindery audit emptytext ~/Downloads --min-chars 1000
41
+
42
+ Library mode pulls the EPUB list (and tags / declared language) from
43
+ metadata.db; directory mode scans every .epub it finds recursively, the
44
+ workflow for checking downloads before they enter the library.
45
+
46
+ Exit codes:
47
+ 0 = clean (THIN empty-text hits are advisory and do not fail the run)
48
+ 1 = a real problem found (foreign content, baked page numbers, empty book,
49
+ OCR-damaged prose) or a scan error
50
+ 2 = setup error (missing DB / library, or no .epub files in directory)
51
+ """
52
+
53
+ import argparse
54
+ import sys
55
+ from pathlib import Path
56
+
57
+ sys.path.insert(0, str(Path(__file__).parent))
58
+ import os
59
+ import re
60
+ import sys
61
+ import zipfile
62
+ from collections import Counter
63
+ from pathlib import Path
64
+ from xml.etree import ElementTree as ET
65
+
66
+ import vir_tui as ui
67
+
68
+ # ----------------------------------------------------------------------------
69
+ # Shared scaffolding
70
+ # ----------------------------------------------------------------------------
71
+
72
+
73
+ def resolve_library_root() -> Path | None:
74
+ """The library root is wherever metadata.db sits: next to this script (the
75
+ copy living inside the library) or the current working directory (running
76
+ the repo copy from inside a library), in that order."""
77
+ for d in (Path(__file__).resolve().parent, Path.cwd()):
78
+ if (d / "metadata.db").is_file():
79
+ return d
80
+ return None
81
+
82
+
83
+ # ANSI colours; suppress when stdout isn't a TTY.
84
+ USE_COLOR = sys.stdout.isatty()
85
+ RED = "\033[31m" if USE_COLOR else ""
86
+ YELLOW = "\033[33m" if USE_COLOR else ""
87
+ GREEN = "\033[32m" if USE_COLOR else ""
88
+ BOLD = "\033[1m" if USE_COLOR else ""
89
+ RESET = "\033[0m" if USE_COLOR else ""
90
+
91
+ CONTAINER_NS = {"c": "urn:oasis:names:tc:opendocument:xmlns:container"}
92
+ OPF_NS = "{http://www.idpf.org/2007/opf}"
93
+
94
+ _PCT = re.compile(r"(?:%[0-9A-Fa-f]{2})+")
95
+
96
+
97
+ def _pct_decode(href: str) -> str:
98
+ """Percent-decode an OPF href (UTF-8). OPF manifest hrefs are IRIs, so a
99
+ reserved char like '!' is written '%21' and a multi-byte char as a run of
100
+ %XX (e.g. 'ö' -> '%C3%B6'); each run must be decoded as bytes together.
101
+ Stdlib-only stand-in for urllib.parse.unquote; invalid escapes stay literal."""
102
+ return _PCT.sub(
103
+ lambda m: bytes.fromhex(m.group(0).replace("%", "")).decode("utf-8", "replace"),
104
+ href,
105
+ )
106
+
107
+
108
+ class Book:
109
+ """A decompressed EPUB: spine documents read once and shared by every
110
+ analyzer. The whole point of the single-pass design lives here."""
111
+
112
+ __slots__ = (
113
+ "_visible",
114
+ "corrupt",
115
+ "docs",
116
+ "lang",
117
+ "names",
118
+ "nav",
119
+ "spine",
120
+ "toc_refs",
121
+ "toc_absent",
122
+ )
123
+
124
+ def __init__(self, spine, nav, lang, docs, names, corrupt, toc_refs, toc_absent):
125
+ self.spine = spine # resolved, in-order, in-archive spine doc paths
126
+ self.nav = nav # the nav document path, or None
127
+ self.lang = lang # declared dc:language (lowercased), or ""
128
+ self.docs = docs # {path: decoded html} for every spine doc
129
+ self.names = names # full archive namelist (for image / marker counts)
130
+ self.corrupt = corrupt # entries whose full read failed (CRC/truncation)
131
+ # Spine-integrity accounting (phase 8): nav/NCX ToC references vs what
132
+ # the archive actually contains. The official Wandering Inn builds
133
+ # ship series-wide ToC manifests (~750 references vs 14-19 real
134
+ # content docs) — reported as `convention`, not flagged.
135
+ self.toc_refs = toc_refs
136
+ self.toc_absent = toc_absent
137
+ self._visible: list[str] | None = None
138
+
139
+ def visible_texts(self) -> list[str]:
140
+ """Rendered text, one entry per spine doc, computed once per book.
141
+
142
+ emptytext and ocr both need it, and under `all` they each used to strip
143
+ tags over the whole book independently: the second pass was pure waste
144
+ in a design whose entire point is touching each EPUB once.
145
+ """
146
+ if self._visible is None:
147
+ self._visible = [_visible_text(self.docs.get(d, "")) for d in self.spine]
148
+ return self._visible
149
+
150
+
151
+ def load_book(path: Path) -> Book:
152
+ """Open an EPUB once: fully read EVERY archive entry (the CRC sweep the
153
+ phase-1 skill used to do by hand), resolve spine + nav + declared
154
+ language, and read every spine document's HTML. Decoded utf-8 with
155
+ replacement (never raises); a corrupted archive entry (bad CRC, truncated
156
+ stream) is recorded in `Book.corrupt` and reads as empty text — the
157
+ corruption verdict, not emptytext, owns that story."""
158
+ with zipfile.ZipFile(path) as z:
159
+ names = z.namelist()
160
+ nameset = set(names)
161
+ corrupt: list[str] = []
162
+ raw: dict[str, bytes] = {}
163
+
164
+ def _read(name: str) -> bytes | None:
165
+ """Fully read one entry (CRC + real decompression, not just the
166
+ central directory's word). A corrupt entry is recorded and its
167
+ content treated as absent — never silently swallowed."""
168
+ if name in raw:
169
+ return raw[name]
170
+ if name in corrupt:
171
+ return None
172
+ try:
173
+ blob = z.read(name)
174
+ except Exception:
175
+ corrupt.append(name)
176
+ return None
177
+ raw[name] = blob
178
+ return blob
179
+
180
+ container = ET.fromstring(_read("META-INF/container.xml"))
181
+ rootfile = container.find(".//c:rootfile", CONTAINER_NS)
182
+ opf_path = rootfile.get("full-path") if rootfile is not None else None
183
+ if not opf_path:
184
+ raise ValueError("container.xml has no rootfile")
185
+ opf = ET.fromstring(_read(opf_path))
186
+ base = os.path.dirname(opf_path)
187
+
188
+ manifest: dict[str, str] = {}
189
+ nav_href: str | None = None
190
+ for it in opf.iter(OPF_NS + "item"):
191
+ item_id, href = it.get("id"), it.get("href")
192
+ if not item_id or not href:
193
+ continue
194
+ manifest[item_id] = href
195
+ if "nav" in (it.get("properties") or ""):
196
+ nav_href = href
197
+
198
+ lang = ""
199
+ for el in opf.iter():
200
+ if el.tag.endswith("}language") and el.text:
201
+ lang = el.text.strip().lower()
202
+ break
203
+
204
+ def full(href: str) -> str:
205
+ # Decode the percent-encoded IRI and drop any #fragment before
206
+ # matching the archive namelist; otherwise a spine doc whose
207
+ # filename has a reserved char (e.g. '!' written '%21') resolves to
208
+ # nothing and the book reads as empty (false EMPTY verdict).
209
+ href = _pct_decode(href.split("#", 1)[0])
210
+ return os.path.normpath(f"{base}/{href}" if base else href).replace(
211
+ "\\", "/"
212
+ )
213
+
214
+ spine: list[str] = []
215
+ for itemref in opf.iter(OPF_NS + "itemref"):
216
+ idref = itemref.get("idref")
217
+ href = manifest.get(idref) if idref else None
218
+ if href and full(href) in nameset:
219
+ spine.append(full(href))
220
+ nav = full(nav_href) if nav_href else None
221
+
222
+ # The single full pass: every archive entry is fully read here (the
223
+ # CRC sweep), so the text analysis below runs on what actually
224
+ # decompressed. Corrupt entries are named, not averaged away.
225
+ for name in names:
226
+ if name.endswith("/"):
227
+ continue
228
+ _read(name)
229
+
230
+ docs: dict[str, str] = {}
231
+ for doc in spine:
232
+ blob = raw.get(doc)
233
+ docs[doc] = blob.decode("utf-8", "replace") if blob is not None else ""
234
+ # ToC reference accounting: every nav/NCX anchor target checked against
235
+ # the archive (manifest items are the spine's own source and already
236
+ # accounted by the spine resolution).
237
+ toc_refs = 0
238
+ toc_absent = 0
239
+
240
+ def _account(href: str) -> None:
241
+ nonlocal toc_refs, toc_absent
242
+ href = href.strip()
243
+ if href.startswith("#"):
244
+ return # in-page anchor, not an archive target
245
+ if re.match(r"^[a-z]+:", href, re.IGNORECASE):
246
+ return # foreign URL scheme, not an archive target
247
+ toc_refs += 1
248
+ if full(href) not in nameset:
249
+ toc_absent += 1
250
+
251
+ nav_html = docs.get(nav) if nav else None
252
+ if nav_html is None and nav:
253
+ # The nav document is usually in the spine, but nothing requires it.
254
+ blob = _read(nav)
255
+ nav_html = blob.decode("utf-8", "replace") if blob is not None else ""
256
+ if nav_html:
257
+ for m in re.finditer(r'href\s*=\s*["\']([^"\']+)', nav_html):
258
+ _account(m.group(1))
259
+ ncx_path = None
260
+ for it in opf.iter(OPF_NS + "item"):
261
+ if (it.get("media-type") or "").lower() == "application/x-dtbncx+xml":
262
+ ncx_path = full(it.get("href") or "")
263
+ break
264
+ if ncx_path:
265
+ blob = _read(ncx_path)
266
+ if blob is not None:
267
+ for m in re.finditer(
268
+ r'<content[^>]+src\s*=\s*["\']([^"\']+)',
269
+ blob.decode("utf-8", "replace"),
270
+ ):
271
+ _account(m.group(1))
272
+
273
+ return Book(spine, nav, lang, docs, names, corrupt, toc_refs, toc_absent)
274
+
275
+
276
+ # ----------------------------------------------------------------------------
277
+ # Analyzer: content (non-English / injected notices)
278
+ # ----------------------------------------------------------------------------
279
+
280
+ CAP = 400_000 # clean chars read per book; ample for a language verdict
281
+
282
+ STYLE_RE = re.compile(r"<(style|script)\b[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
283
+ TAG_RE = re.compile(r"<[^>]+>")
284
+ WS_RE = re.compile(r"\s+")
285
+ WORD_RE = re.compile(r"[a-zA-ZàâäéèêëïîôöùûüçñáíóúãõßÀ-ÿ']+")
286
+ SIGNATURE_RE = re.compile(r"importknig|книжный импорт|knizhny", re.IGNORECASE)
287
+
288
+ # Distinctive stopword sets. Book-length text makes the vote unambiguous;
289
+ # the small EN/IT overlap on "i" etc. is swamped by the rest.
290
+ STOPWORDS: dict[str, set[str]] = {
291
+ "en": set(
292
+ [
293
+ "the",
294
+ "of",
295
+ "and",
296
+ "to",
297
+ "a",
298
+ "in",
299
+ "that",
300
+ "is",
301
+ "was",
302
+ "for",
303
+ "it",
304
+ "with",
305
+ "as",
306
+ "his",
307
+ "on",
308
+ "be",
309
+ "at",
310
+ "by",
311
+ "he",
312
+ "this",
313
+ "had",
314
+ "not",
315
+ "are",
316
+ "but",
317
+ "from",
318
+ "or",
319
+ "have",
320
+ "an",
321
+ "they",
322
+ "which",
323
+ "one",
324
+ "you",
325
+ "were",
326
+ "her",
327
+ "all",
328
+ "she",
329
+ "there",
330
+ "would",
331
+ "their",
332
+ ]
333
+ ),
334
+ "de": set(
335
+ [
336
+ "der",
337
+ "die",
338
+ "und",
339
+ "in",
340
+ "den",
341
+ "von",
342
+ "zu",
343
+ "das",
344
+ "mit",
345
+ "sich",
346
+ "des",
347
+ "auf",
348
+ "für",
349
+ "ist",
350
+ "im",
351
+ "dem",
352
+ "nicht",
353
+ "ein",
354
+ "eine",
355
+ "als",
356
+ "auch",
357
+ "es",
358
+ "an",
359
+ "werden",
360
+ "aus",
361
+ "er",
362
+ "hat",
363
+ "dass",
364
+ "sie",
365
+ "nach",
366
+ "wird",
367
+ "bei",
368
+ "einer",
369
+ "um",
370
+ ]
371
+ ),
372
+ "fr": set(
373
+ [
374
+ "le",
375
+ "la",
376
+ "les",
377
+ "de",
378
+ "des",
379
+ "un",
380
+ "une",
381
+ "et",
382
+ "en",
383
+ "dans",
384
+ "que",
385
+ "qui",
386
+ "pour",
387
+ "pas",
388
+ "sur",
389
+ "au",
390
+ "avec",
391
+ "ce",
392
+ "il",
393
+ "ne",
394
+ "se",
395
+ "plus",
396
+ "par",
397
+ "je",
398
+ "nous",
399
+ "vous",
400
+ "est",
401
+ "son",
402
+ "ses",
403
+ "aux",
404
+ ]
405
+ ),
406
+ "es": set(
407
+ [
408
+ "el",
409
+ "la",
410
+ "los",
411
+ "las",
412
+ "de",
413
+ "un",
414
+ "una",
415
+ "y",
416
+ "en",
417
+ "que",
418
+ "no",
419
+ "se",
420
+ "con",
421
+ "por",
422
+ "para",
423
+ "es",
424
+ "su",
425
+ "lo",
426
+ "como",
427
+ "más",
428
+ "pero",
429
+ "sus",
430
+ "le",
431
+ "ya",
432
+ "o",
433
+ "este",
434
+ "sí",
435
+ "porque",
436
+ "esta",
437
+ "entre",
438
+ ]
439
+ ),
440
+ "it": set(
441
+ [
442
+ "il",
443
+ "lo",
444
+ "la",
445
+ "i",
446
+ "gli",
447
+ "le",
448
+ "di",
449
+ "un",
450
+ "uno",
451
+ "una",
452
+ "e",
453
+ "che",
454
+ "non",
455
+ "per",
456
+ "con",
457
+ "su",
458
+ "come",
459
+ "più",
460
+ "ma",
461
+ "anche",
462
+ "da",
463
+ "sono",
464
+ "mi",
465
+ "si",
466
+ "nel",
467
+ "alla",
468
+ "dei",
469
+ "delle",
470
+ ]
471
+ ),
472
+ "pt": set(
473
+ [
474
+ "o",
475
+ "a",
476
+ "os",
477
+ "as",
478
+ "de",
479
+ "um",
480
+ "uma",
481
+ "e",
482
+ "que",
483
+ "do",
484
+ "da",
485
+ "em",
486
+ "não",
487
+ "se",
488
+ "com",
489
+ "por",
490
+ "para",
491
+ "mais",
492
+ "mas",
493
+ "como",
494
+ "ao",
495
+ "dos",
496
+ "das",
497
+ "na",
498
+ "no",
499
+ "à",
500
+ "seu",
501
+ ]
502
+ ),
503
+ "nl": set(
504
+ [
505
+ "de",
506
+ "het",
507
+ "een",
508
+ "en",
509
+ "van",
510
+ "te",
511
+ "dat",
512
+ "die",
513
+ "in",
514
+ "is",
515
+ "op",
516
+ "ik",
517
+ "niet",
518
+ "met",
519
+ "zijn",
520
+ "er",
521
+ "maar",
522
+ "om",
523
+ "ook",
524
+ "als",
525
+ "voor",
526
+ "naar",
527
+ "dan",
528
+ "zou",
529
+ "hij",
530
+ "heeft",
531
+ ]
532
+ ),
533
+ }
534
+
535
+
536
+ def script_of(codepoint: int) -> str | None:
537
+ if 0x0400 <= codepoint <= 0x04FF:
538
+ return "Cyrillic"
539
+ if 0x4E00 <= codepoint <= 0x9FFF:
540
+ return "CJK-Han"
541
+ if 0x3040 <= codepoint <= 0x30FF:
542
+ return "Japanese-kana"
543
+ if 0xAC00 <= codepoint <= 0xD7A3:
544
+ return "Korean"
545
+ if 0x0600 <= codepoint <= 0x06FF:
546
+ return "Arabic"
547
+ if 0x0370 <= codepoint <= 0x03FF:
548
+ return "Greek"
549
+ if 0x0590 <= codepoint <= 0x05FF:
550
+ return "Hebrew"
551
+ if 0x0900 <= codepoint <= 0x097F:
552
+ return "Devanagari"
553
+ return None
554
+
555
+
556
+ def _content_clean(html: str) -> str:
557
+ html = STYLE_RE.sub(" ", html)
558
+ return WS_RE.sub(" ", TAG_RE.sub(" ", html))
559
+
560
+
561
+ def analyze_content(book: Book) -> dict:
562
+ """Language / script signal over the (pre-read) spine."""
563
+ docs = book.spine
564
+ parts: list[str] = []
565
+ total = 0
566
+ for doc in docs:
567
+ if total >= CAP:
568
+ break
569
+ text = _content_clean(book.docs.get(doc, ""))
570
+ parts.append(text)
571
+ total += len(text)
572
+ if docs and docs[-1] not in docs[: len(parts)]:
573
+ parts.append(_content_clean(book.docs.get(docs[-1], "")))
574
+
575
+ text = " ".join(parts)
576
+ # Count scripts over the first 250k letters in a single pass, instead of
577
+ # materializing a list of every letter's codepoint in a book-length string.
578
+ scripts: Counter = Counter()
579
+ total_letters = 0
580
+ for c in text:
581
+ if c.isalpha():
582
+ total_letters += 1
583
+ s = script_of(ord(c))
584
+ if s:
585
+ scripts[s] += 1
586
+ if total_letters >= 250_000:
587
+ break
588
+ nonlatin = sum(scripts.values())
589
+ total_letters = total_letters or 1
590
+ words = WORD_RE.findall(text.lower())[:5000]
591
+ ratios = {
592
+ code: (sum(w in stops for w in words) / len(words) if words else 0.0)
593
+ for code, stops in STOPWORDS.items()
594
+ }
595
+ best = max(ratios, key=lambda c: ratios[c])
596
+ return {
597
+ "lang": book.lang,
598
+ "scripts": dict(scripts),
599
+ "nonlatin": nonlatin,
600
+ "nonlatin_frac": nonlatin / total_letters,
601
+ "ratios": ratios,
602
+ "best": best,
603
+ "nwords": len(words),
604
+ "signature": bool(SIGNATURE_RE.search(text)),
605
+ }
606
+
607
+
608
+ def findings(r: dict) -> list[tuple[str, str]]:
609
+ """Classify a content result into [(category, detail)]; empty = English and clean."""
610
+ out: list[tuple[str, str]] = []
611
+ if r["nonlatin"] >= 150 and r["nonlatin_frac"] > 0.02:
612
+ top = max(r["scripts"], key=lambda s: r["scripts"][s])
613
+ out.append(
614
+ (
615
+ "NON-LATIN SCRIPT",
616
+ f"{r['nonlatin_frac'] * 100:.0f}% {top} ({r['nonlatin']} non-Latin letters)",
617
+ )
618
+ )
619
+ if (
620
+ r["best"] != "en"
621
+ and r["nwords"] >= 400
622
+ and r["ratios"][r["best"]] > 0.06
623
+ and r["ratios"][r["best"]] > 1.3 * r["ratios"]["en"]
624
+ ):
625
+ top3 = ", ".join(
626
+ f"{c}={v:.2f}"
627
+ for c, v in sorted(r["ratios"].items(), key=lambda x: -x[1])[:3]
628
+ )
629
+ out.append(("LATIN-SCRIPT FOREIGN", f"looks {r['best'].upper()} [{top3}]"))
630
+ if r["signature"]:
631
+ out.append(
632
+ ("INJECTION SIGNATURE", "importknig / Книжный импорт signature present")
633
+ )
634
+ return out
635
+
636
+
637
+ def scan_content(path: Path) -> dict:
638
+ """Convenience: load + analyze a single file (used by tests / ad-hoc runs)."""
639
+ return analyze_content(load_book(path))
640
+
641
+
642
+ # ----------------------------------------------------------------------------
643
+ # Analyzer: pagenumbers (print page numbers baked into body text)
644
+ # ----------------------------------------------------------------------------
645
+
646
+ INT_RE = re.compile(r"\d{1,4}$")
647
+ ROMAN_RE = re.compile(r"[ivxlcdm]{2,7}$", re.IGNORECASE)
648
+ # Block-level elements we track to reconstruct reading order.
649
+ BLOCK_TAGS = {
650
+ "p",
651
+ "div",
652
+ "h1",
653
+ "h2",
654
+ "h3",
655
+ "h4",
656
+ "h5",
657
+ "h6",
658
+ "li",
659
+ "td",
660
+ "th",
661
+ "blockquote",
662
+ "section",
663
+ "caption",
664
+ "figcaption",
665
+ }
666
+
667
+ # Tuning, validated by hand against a 3,897-EPUB library (every flagged book and
668
+ # the full n>=2 tail inspected). At these values the detector flagged 21 books,
669
+ # all true positives; the false-positive tail (experimental footnote-poems,
670
+ # scraped web-serial vote counts, placeholder section labels) all fell under
671
+ # MIN_BAKED_HITS or MIN_SPAN. Lowering MIN_BAKED_HITS to catch the few remaining
672
+ # 3-4 hit books (a localized stray-number patch) starts admitting those.
673
+ PROSE_MIN = 120 # a neighbour this long counts as a prose paragraph
674
+ RUNHEAD_MIN_REPEAT = 8 # a short block repeated this often is a running header
675
+ RUNHEAD_MAX_LEN = 60 # running headers are short
676
+ MIN_BAKED_HITS = 5 # below this, a handful of hits is too often coincidence
677
+ MIN_SPAN = 0.10 # flagged numbers must cover this fraction of the book (drops
678
+ # localized clusters: footnote-poems, scraped comment sections)
679
+ MIN_RUN = 1 # ascending run is informative but not gated; the baked test already
680
+ # requires genuine sentence interruption, so a short run is not disqualifying
681
+
682
+
683
+ def roman_value(s: str) -> int | None:
684
+ vals = {"i": 1, "v": 5, "x": 10, "l": 50, "c": 100, "d": 500, "m": 1000}
685
+ total = 0
686
+ s = s.lower()
687
+ for i, c in enumerate(s):
688
+ if c not in vals:
689
+ return None
690
+ v = vals[c]
691
+ total += -v if (i + 1 < len(s) and vals[s[i + 1]] > v) else v
692
+ return total or None
693
+
694
+
695
+ def number_value(text: str) -> int | None:
696
+ """A bare page-number-ish value (1-9999 arabic, or a roman numeral), else None."""
697
+ if INT_RE.fullmatch(text):
698
+ return int(text)
699
+ if ROMAN_RE.fullmatch(text):
700
+ return roman_value(text)
701
+ return None
702
+
703
+
704
+ class _Blocks:
705
+ """Minimal block extractor over html.parser; tracks the innermost block tag
706
+ so each emitted (tag, text) pair is one rendered block in reading order.
707
+
708
+ img_before[i] is True when an image (img/svg) appeared between the last
709
+ text before blocks[i] and blocks[i]'s own first text. Used by the ocr
710
+ analyzer to clear paragraphs interrupted by a rendered figure (an inline
711
+ formula or card diagram reads as a mid-sentence split otherwise).
712
+ in_quote[i] is True when blocks[i] was emitted inside a <blockquote>
713
+ (a display quotation legitimately starts and ends mid-sentence). Both are
714
+ ocr-only; pagenumbers ignores them."""
715
+
716
+ def __init__(self):
717
+ from html.parser import HTMLParser
718
+
719
+ outer = self
720
+
721
+ class _P(HTMLParser):
722
+ def __init__(self):
723
+ super().__init__(convert_charrefs=True)
724
+ self.stack: list[str] = []
725
+ self.buf: list[str] = []
726
+ self.img_gap = False # image seen since the last text content
727
+ self.block_img = False # img_gap captured at this block's first text
728
+ self.has_text = False
729
+
730
+ def handle_starttag(self, tag, attrs):
731
+ del attrs
732
+ if tag in ("img", "image", "svg"):
733
+ self.img_gap = True
734
+ if tag in BLOCK_TAGS:
735
+ outer._flush(self)
736
+ self.stack.append(tag)
737
+
738
+ def handle_endtag(self, tag):
739
+ if tag in BLOCK_TAGS:
740
+ outer._flush(self)
741
+ if self.stack:
742
+ self.stack.pop()
743
+
744
+ def handle_data(self, data):
745
+ if data.strip():
746
+ if not self.has_text:
747
+ self.block_img = self.img_gap
748
+ self.has_text = True
749
+ self.img_gap = False
750
+ self.buf.append(data)
751
+
752
+ self.blocks: list[tuple[str, str]] = []
753
+ self.img_before: list[bool] = []
754
+ self.in_quote: list[bool] = []
755
+ self._parser = _P()
756
+
757
+ def _flush(self, parser):
758
+ text = "".join(parser.buf).strip()
759
+ parser.buf.clear()
760
+ if text:
761
+ tag = parser.stack[-1] if parser.stack else "?"
762
+ self.blocks.append((tag, text))
763
+ self.img_before.append(parser.block_img)
764
+ self.in_quote.append("blockquote" in parser.stack)
765
+ parser.has_text = False
766
+ parser.block_img = False
767
+
768
+ def feed(self, html: str):
769
+ self._parser.feed(html)
770
+ self._parser.close()
771
+ self._flush(self._parser)
772
+
773
+
774
+ def analyze_pagenumbers(book: Book) -> dict:
775
+ """Score baked-in page numbers over the (pre-read) spine, skipping nav."""
776
+ blocks: list[tuple[str, str]] = []
777
+ for doc in book.spine:
778
+ if book.nav and doc == book.nav:
779
+ continue
780
+ parser = _Blocks()
781
+ try:
782
+ parser.feed(book.docs.get(doc, ""))
783
+ except Exception:
784
+ continue
785
+ li = sum(1 for t, _ in parser.blocks if t == "li")
786
+ # a doc that is mostly <li> is a TOC / page-list, not body text
787
+ if parser.blocks and li / len(parser.blocks) > 0.5:
788
+ continue
789
+ blocks.extend(parser.blocks)
790
+
791
+ text_len = sum(len(t) for _, t in blocks) or 1
792
+
793
+ # Running headers / footers / watermarks: short blocks repeated many times.
794
+ freq = Counter(
795
+ t
796
+ for t in (b[1] for b in blocks)
797
+ if len(t) <= RUNHEAD_MAX_LEN and number_value(t) is None
798
+ )
799
+ runheads = {t for t, c in freq.items() if c >= RUNHEAD_MIN_REPEAT}
800
+
801
+ hits: list[tuple[int, int]] = [] # (char offset, value)
802
+ examples: list[dict] = []
803
+ offset = 0
804
+ for i, (tag, text) in enumerate(blocks):
805
+ if tag in ("p", "div"):
806
+ v = number_value(text)
807
+ # 1500-2099 are almost always years (chronologies, dated chapters),
808
+ # not page numbers; a real page count rarely reaches them.
809
+ if v is not None and not (1500 <= v <= 2099):
810
+ ptag, ptxt = blocks[i - 1] if i > 0 else ("", "")
811
+ ntag, ntxt = blocks[i + 1] if i + 1 < len(blocks) else ("", "")
812
+ prose_prev = ptag in ("p", "div") and len(ptxt) > PROSE_MIN
813
+ prose_next = ntag in ("p", "div") and len(ntxt) > PROSE_MIN
814
+ if prose_prev or prose_next:
815
+ prev_runhead = ptxt in runheads
816
+ next_runhead = ntxt in runheads
817
+ word_split = (
818
+ bool(ptxt) and ptxt[-1] == "-" and ptxt[-2:-1].isalpha()
819
+ )
820
+ lower_cont = bool(ntxt) and ntxt[0].islower()
821
+ # the previous body paragraph is unfinished (ends mid-word /
822
+ # mid-clause), so the number is wedged into a live sentence.
823
+ prev_unfinished = prose_prev and (
824
+ ptxt[-1].islower() or ptxt[-1] == ","
825
+ )
826
+ baked = (
827
+ word_split
828
+ or lower_cont
829
+ or (prev_unfinished and (prose_next or next_runhead))
830
+ or (prev_runhead and next_runhead)
831
+ )
832
+ if baked:
833
+ hits.append((offset, v))
834
+ if len(examples) < 8:
835
+ examples.append(
836
+ {"v": text, "prev": ptxt[-60:], "next": ntxt[:60]}
837
+ )
838
+ offset += len(text)
839
+
840
+ vals = [v for _, v in hits]
841
+ if hits:
842
+ span = (hits[-1][0] - hits[0][0]) / text_len
843
+ else:
844
+ span = 0.0
845
+ run = best = 1 if vals else 0
846
+ for i in range(1, len(vals)):
847
+ if vals[i] - vals[i - 1] in (1, 2):
848
+ best += 1
849
+ run = max(run, best)
850
+ else:
851
+ best = 1
852
+ return {
853
+ "n_hits": len(hits),
854
+ "span": span,
855
+ "run": run,
856
+ "watermark": any(
857
+ re.search(r"download|boykma|\.com\b", h, re.IGNORECASE) for h in runheads
858
+ ),
859
+ "examples": examples,
860
+ }
861
+
862
+
863
+ def is_defective(r: dict) -> bool:
864
+ return (
865
+ r["n_hits"] >= MIN_BAKED_HITS and r["span"] >= MIN_SPAN and r["run"] >= MIN_RUN
866
+ )
867
+
868
+
869
+ def scan_pagenumbers(path: Path) -> dict:
870
+ return analyze_pagenumbers(load_book(path))
871
+
872
+
873
+ # ----------------------------------------------------------------------------
874
+ # Analyzer: emptytext (empty / no-body-text stubs)
875
+ # ----------------------------------------------------------------------------
876
+
877
+ DEFAULT_MIN_CHARS = 2000 # at or below this: EMPTY (real defect)
878
+ DEFAULT_THIN_CHARS = 20000 # below this: THIN (advisory review)
879
+ # Monolithic-document floor (phase-1 skill: "roughly 300-500k" chars is where
880
+ # readers start refusing). A single content doc at or above this flags.
881
+ DEFAULT_MAX_DOC_CHARS = 300_000
882
+
883
+ IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg")
884
+ BOOKMATE_MARKERS = ("bookmate.css", "calibre_bookmarks.txt")
885
+
886
+ # Partial / placeholder exports: a DRM-locked or sample export leaves most
887
+ # chapters as the same tiny "content unavailable" placeholder, so the book
888
+ # validates and clears the total-char floor on the strength of one or two real
889
+ # chapters (the whole-book char count is fooled). Detect by a known signature,
890
+ # or by the same short stub repeated across a large fraction of the spine.
891
+ # Seen cases: BookShout ("something went wrong loading... bookshout.com").
892
+ _PLACEHOLDER_SIG = re.compile(
893
+ r"bookshout|something went wrong loading|failed to load|"
894
+ r"(content|page|book)\s+(is\s+)?(not available|unavailable|could not be loaded)",
895
+ re.IGNORECASE,
896
+ )
897
+ PLACEHOLDER_STUB_MIN = 12 # ignore blank / trivial spine docs
898
+ PLACEHOLDER_STUB_MAX = 600 # a placeholder stub is short
899
+ PLACEHOLDER_MIN_REPEAT = 3 # the same stub across at least this many spine docs
900
+ PLACEHOLDER_MIN_FRAC = 0.30 # ...and at least this fraction of the spine
901
+
902
+ _SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b.*?</\1>", re.IGNORECASE | re.DOTALL)
903
+ _TAG_RE = re.compile(r"<[^>]+>")
904
+ _WS_RE = re.compile(r"\s+")
905
+
906
+
907
+ def _visible_text(html: str) -> str:
908
+ """Rendered text: drop script/style, strip tags, decode entities, collapse
909
+ whitespace."""
910
+ from html import unescape
911
+
912
+ html = _SCRIPT_STYLE_RE.sub(" ", html)
913
+ text = _TAG_RE.sub(" ", html)
914
+ text = unescape(text)
915
+ return _WS_RE.sub(" ", text).strip()
916
+
917
+
918
+ def _visible_chars(html: str) -> int:
919
+ return len(_visible_text(html))
920
+
921
+
922
+ def analyze_emptytext(book: Book) -> dict:
923
+ """Count visible text across the (pre-read) spine and gather triage signals."""
924
+ texts = book.visible_texts()
925
+ chars = sum(len(t) for t in texts)
926
+ images = sum(1 for n in book.names if n.lower().endswith(IMAGE_EXTS))
927
+ bookmate = any(any(m in n.lower() for m in BOOKMATE_MARKERS) for n in book.names)
928
+
929
+ # Partial / placeholder export: a known DRM signature anywhere, or the same
930
+ # short stub repeated across a large fraction of the spine (most chapters
931
+ # replaced by an identical "content unavailable" notice). Blank docs are
932
+ # excluded by PLACEHOLDER_STUB_MIN so well-made books full of small section
933
+ # dividers (each with distinct text) do not trip it.
934
+ sig = any(_PLACEHOLDER_SIG.search(t) for t in texts)
935
+ stubs = Counter(
936
+ t for t in texts if PLACEHOLDER_STUB_MIN <= len(t) <= PLACEHOLDER_STUB_MAX
937
+ )
938
+ stub_n = stubs.most_common(1)[0][1] if stubs else 0
939
+ spine_n = max(1, len(book.spine))
940
+ repeated = (
941
+ stub_n >= PLACEHOLDER_MIN_REPEAT and stub_n / spine_n >= PLACEHOLDER_MIN_FRAC
942
+ )
943
+ return {
944
+ "chars": chars,
945
+ "spine_len": len(book.spine),
946
+ "images": images,
947
+ "bookmate": bookmate,
948
+ "placeholder": bool(sig or repeated),
949
+ "placeholder_sig": sig,
950
+ "stub_docs": stub_n,
951
+ "corrupt_n": len(book.corrupt),
952
+ "corrupt_first": book.corrupt[0] if book.corrupt else "",
953
+ }
954
+
955
+
956
+ def classify(r: dict, min_chars: int, thin_chars: int) -> str:
957
+ # A corrupt archive entry reads as empty text; reporting EMPTY here would
958
+ # be the right alarm for the wrong disease. The corruption verdict owns it.
959
+ if r.get("corrupt_n"):
960
+ return "CORRUPT"
961
+ if r["chars"] <= min_chars:
962
+ return "EMPTY"
963
+ if r.get("placeholder"):
964
+ return "PARTIAL"
965
+ if r["chars"] < thin_chars:
966
+ return "THIN"
967
+ return "OK"
968
+
969
+
970
+ def _empty_detail(r: dict) -> str:
971
+ bits = [f"{r['chars']} chars", f"spine {r['spine_len']}", f"{r['images']} images"]
972
+ if r.get("corrupt_n"):
973
+ bits.append(f"corrupt:{r['corrupt_n']} (first: {r['corrupt_first']})")
974
+ if r["bookmate"]:
975
+ bits.append("bookmate")
976
+ if r.get("placeholder"):
977
+ if r.get("placeholder_sig"):
978
+ bits.append("partial: DRM/placeholder signature")
979
+ else:
980
+ bits.append(
981
+ f"partial: {r['stub_docs']}/{r['spine_len']} identical stub docs"
982
+ )
983
+ return ", ".join(bits)
984
+
985
+
986
+ def scan_emptytext(path: Path) -> dict:
987
+ return analyze_emptytext(load_book(path))
988
+
989
+
990
+ # ----------------------------------------------------------------------------
991
+ # Analyzer: monolithic (one spine doc that is far too large)
992
+ # ----------------------------------------------------------------------------
993
+
994
+
995
+ def analyze_monolithic(book: Book) -> dict:
996
+ """Find the largest single content document, in characters.
997
+
998
+ Monolithic documents are invisible to emptytext (whole-book volume, not
999
+ per-doc shape) and to epubcheck (which never sees renderer memory limits):
1000
+ a "clean" 30M-char dictionary that will not render past a point on real
1001
+ readers. Reuses the shared visible_texts() cache, so this adds no second
1002
+ decompression pass.
1003
+ """
1004
+ texts = book.visible_texts()
1005
+ worst_i = max(range(len(texts)), key=lambda i: len(texts[i]), default=None)
1006
+ return {
1007
+ "max_doc_chars": len(texts[worst_i]) if worst_i is not None else 0,
1008
+ "worst_doc": book.spine[worst_i] if worst_i is not None else "",
1009
+ "spine_len": len(book.spine),
1010
+ }
1011
+
1012
+
1013
+ def is_monolithic(r: dict, max_doc_chars: int) -> bool:
1014
+ """Flag only at or above the threshold; high-but-under stays silent
1015
+ (advisory silence, like THIN)."""
1016
+ return r["max_doc_chars"] >= max_doc_chars
1017
+
1018
+
1019
+ def scan_monolithic(path: Path) -> dict:
1020
+ return analyze_monolithic(load_book(path))
1021
+
1022
+
1023
+ # ----------------------------------------------------------------------------
1024
+ # Analyzer: archive corruption (always on; owns the "empty body" story)
1025
+ # ----------------------------------------------------------------------------
1026
+
1027
+
1028
+ def analyze_corrupt(book: Book) -> dict:
1029
+ """Archive entries whose full read failed (bad CRC, truncated stream).
1030
+
1031
+ Reported as its own verdict in every mode — a corrupt entry decompresses
1032
+ to nothing, and letting emptytext call that EMPTY mislabels a damaged
1033
+ archive as a content-less stub (the phase-1 re-source advice that follows
1034
+ from EMPTY would then aim at the wrong disease).
1035
+ """
1036
+ return {"n": len(book.corrupt), "first": book.corrupt[0] if book.corrupt else ""}
1037
+
1038
+
1039
+ def _corrupt_verdict(r: dict) -> tuple[bool, str, list[str]]:
1040
+ return (
1041
+ True,
1042
+ "CORRUPT",
1043
+ [f"corrupt:{r['n']} (first: {r['first']}) — damaged archive; re-source"],
1044
+ )
1045
+
1046
+
1047
+ # ----------------------------------------------------------------------------
1048
+ # Analyzer: spine integrity (ToC bloat vs a true fragment)
1049
+ # ----------------------------------------------------------------------------
1050
+
1051
+
1052
+ def spine_integrity(book: Book) -> dict:
1053
+ """Classify absent ToC/NCX targets.
1054
+
1055
+ The official Wandering Inn builds ship series-wide ToC manifests (~750
1056
+ references vs 14-19 real content docs): `convention` when the absent
1057
+ count is ~= the reference count and the present docs' chapter span is
1058
+ consecutive — reported, never flagged. A broken span means the book is
1059
+ missing part of itself: `fragment`, flagged. Unnumbered or absent==0
1060
+ cases are `ok`/`unknown` (silent or advisory).
1061
+ """
1062
+ absent, refs = book.toc_absent, book.toc_refs
1063
+ if absent == 0:
1064
+ return {"class": "ok", "refs": refs, "absent": 0}
1065
+ nums = []
1066
+ for doc in book.spine:
1067
+ found = re.findall(r"\d+", doc.rsplit("/", 1)[-1])
1068
+ if not found:
1069
+ return {"class": "unknown", "refs": refs, "absent": absent}
1070
+ nums.append(int(found[-1]))
1071
+ nums.sort()
1072
+ span = (
1073
+ "consecutive" if nums == list(range(nums[0], nums[0] + len(nums))) else "broken"
1074
+ )
1075
+ if refs >= 20 and absent >= refs * 0.9 and span == "consecutive":
1076
+ return {"class": "convention", "refs": refs, "absent": absent}
1077
+ if span == "broken":
1078
+ return {"class": "fragment", "refs": refs, "absent": absent}
1079
+ return {"class": "unknown", "refs": refs, "absent": absent}
1080
+
1081
+
1082
+ def _spine_verdict(r: dict) -> tuple[bool, str, list[str]]:
1083
+ if r["class"] == "fragment":
1084
+ return (
1085
+ True,
1086
+ "FRAGMENT",
1087
+ [f"toc {r['refs']} refs, {r['absent']} absent — span broken"],
1088
+ )
1089
+ if r["class"] == "convention":
1090
+ return (
1091
+ False,
1092
+ "ADVISORY",
1093
+ [
1094
+ f"toc {r['refs']} refs, {r['absent']} absent (series-wide manifest convention)"
1095
+ ],
1096
+ )
1097
+ return (
1098
+ False,
1099
+ "ADVISORY",
1100
+ [f"toc {r['refs']} refs, {r['absent']} absent (unjudgeable span)"],
1101
+ )
1102
+
1103
+
1104
+ # ----------------------------------------------------------------------------
1105
+ # Analyzer: ocr (OCR/conversion-damaged prose)
1106
+ # ----------------------------------------------------------------------------
1107
+
1108
+ # Tuning, validated by hand against a 4,605-EPUB library: every book passing
1109
+ # the joint gate was inspected via its split examples. At these values the
1110
+ # detector flagged 105 books; 104 were confirmed damage (line-wrap and
1111
+ # page-break splits, double-spaced OCR text, running headers wedged into
1112
+ # sentences), with one borderline residue (a book whose display quotes are
1113
+ # publisher-styled plain <p>s, indistinguishable from damage without CSS).
1114
+ # The motivating case: a damaged Jingo EPUB measured 80 mid-sentence splits
1115
+ # where a clean edition of the same text measured 0.
1116
+ #
1117
+ # Known misses, by design: damage whose signature is word TRUNCATION ("which
1118
+ # bel[ong] to western Spain") or whitespace corruption rather than paragraph
1119
+ # splitting, and low-grade split damage that lands in the same func_frac band
1120
+ # as literary stream-of-consciousness (Gulag Archipelago at 0.243 vs. The
1121
+ # Sound and the Fury at 0.242: no threshold separates them).
1122
+ OCR_MIN_PARAS = 50 # below this the rate is noise, not a signal
1123
+ OCR_MIN_SPLITS = 10 # absolute floor; a handful of splits is coincidence
1124
+ OCR_FLAG_RATE = 0.010 # splits per prose paragraph; FLAG at or above
1125
+ OCR_FUNC_MIN = 0.25 # see func_frac below
1126
+
1127
+ # A split only counts when one side is a substantial prose block (reuses the
1128
+ # pagenumbers notion of prose), so verse and dialogue beats -- short lines that
1129
+ # legitimately end unpunctuated and start lowercase -- do not accumulate.
1130
+ _OCR_PROSE_MIN = PROSE_MIN
1131
+
1132
+ # Residual false-positive guards, each from a class found in the validation
1133
+ # sweep. A back-of-book index rendered as <p> blocks reads as a wall of
1134
+ # unpunctuated lowercase runs ("See also" is its near-universal signature).
1135
+ # An epistolary sign-off is a dangling sub-40-char fragment with no terminal
1136
+ # punctuation ("in which hope I rest," / "respected sir,"). A display equation
1137
+ # set as text is mostly non-alphabetic ("the equation" / "y2 + y = x3 - x ?")
1138
+ # and renders fine on its own line.
1139
+ _OCR_INDEX_RE = re.compile(r"\bSee also\b")
1140
+ _OCR_INDEX_MIN_BLOCKS = 3
1141
+ _OCR_TAIL_MIN = 40
1142
+ _OCR_TERMINALS = ".!?\"'’”…)"
1143
+ _OCR_ALPHA_MIN = 0.5
1144
+
1145
+
1146
+ def _alpha_density(text: str) -> float:
1147
+ chars = [c for c in text if not c.isspace()]
1148
+ if not chars:
1149
+ return 0.0
1150
+ return sum(c.isalpha() for c in chars) / len(chars)
1151
+
1152
+
1153
+ # The style-vs-damage discriminator. Deliberately unpunctuated literary prose
1154
+ # (Fosse's Septology, Evaristo's "Girl, Woman, Other", Kingsnorth's "The Wake")
1155
+ # racks up enormous split rates, but its paragraphs end at CLAUSE boundaries;
1156
+ # conversion damage splits at line-wrap/page-break positions, so the fragment
1157
+ # ends on a function word ("sat the disembodied" / "heads who were..."). On the
1158
+ # reference library, style books measured func_frac <= 0.11 and every hand-
1159
+ # confirmed damage case >= 0.26, so 0.25 separates cleanly. A small closed
1160
+ # function-word set, same spirit as the content analyzer's stopword votes --
1161
+ # not a dictionary.
1162
+ OCR_FUNC_WORDS = frozenset(
1163
+ [
1164
+ "the",
1165
+ "a",
1166
+ "an",
1167
+ "and",
1168
+ "or",
1169
+ "but",
1170
+ "of",
1171
+ "to",
1172
+ "in",
1173
+ "on",
1174
+ "at",
1175
+ "by",
1176
+ "with",
1177
+ "for",
1178
+ "from",
1179
+ "as",
1180
+ "that",
1181
+ "this",
1182
+ "these",
1183
+ "those",
1184
+ "his",
1185
+ "her",
1186
+ "their",
1187
+ "its",
1188
+ "my",
1189
+ "your",
1190
+ "our",
1191
+ "was",
1192
+ "were",
1193
+ "is",
1194
+ "are",
1195
+ "be",
1196
+ "been",
1197
+ "being",
1198
+ "had",
1199
+ "has",
1200
+ "have",
1201
+ "he",
1202
+ "she",
1203
+ "they",
1204
+ "it",
1205
+ "we",
1206
+ "you",
1207
+ "i",
1208
+ "not",
1209
+ "no",
1210
+ "so",
1211
+ "if",
1212
+ "when",
1213
+ "than",
1214
+ "then",
1215
+ "who",
1216
+ "whom",
1217
+ "which",
1218
+ "into",
1219
+ "onto",
1220
+ "over",
1221
+ "under",
1222
+ "between",
1223
+ "through",
1224
+ "during",
1225
+ "before",
1226
+ "after",
1227
+ "above",
1228
+ "below",
1229
+ "up",
1230
+ "down",
1231
+ "out",
1232
+ "off",
1233
+ "very",
1234
+ "more",
1235
+ "most",
1236
+ "some",
1237
+ "any",
1238
+ "each",
1239
+ "every",
1240
+ "either",
1241
+ "neither",
1242
+ ]
1243
+ )
1244
+
1245
+ # En-dash embedded inside a word: OCR reads a hyphen as U+2013 ("bottom–feedin'").
1246
+ EN_DASH_WORD_RE = re.compile(r"[A-Za-z]–[A-Za-z]")
1247
+ # Doubled opening quote: an opening single quote re-recognized before a
1248
+ # dialogue contraction ("' 'Course"). Straight or curly. The first quote must
1249
+ # follow whitespace: a closing quote hugs the word before it, so this stays
1250
+ # blind to legitimate close-then-open sequences ("...said.' 'No...") that
1251
+ # single-quote-dialogue books produce constantly.
1252
+ DOUBLED_QUOTE_RE = re.compile(r"(?<=\s)['‘]\s+['‘’]\w")
1253
+ # A space-stripped proper-noun compound: CamelCase with 2+ humps
1254
+ # ("AnkhMorpork"); only damage when the hyphenated form also appears.
1255
+ _CAMEL_RE = re.compile(r"\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\b")
1256
+ _HUMP_RE = re.compile(r"(?<=[a-z])(?=[A-Z])")
1257
+
1258
+
1259
+ def analyze_brokentags(book: Book) -> dict:
1260
+ import re
1261
+
1262
+ hits = []
1263
+ pattern = re.compile(
1264
+ r"(?<!<)(?<!&lt;)/[a-zA-Z]+&gt;|&lt;/(?:p|div|span|h[1-6]|i|em|b|strong)&gt;",
1265
+ re.IGNORECASE,
1266
+ )
1267
+ for doc in book.spine:
1268
+ text = book.docs.get(doc, "")
1269
+ for match in pattern.finditer(text):
1270
+ hits.append(f"{doc}: {match.group(0)}")
1271
+ if not hits:
1272
+ return {}
1273
+ return {"hits": hits, "summary": f"{len(hits)} broken tags found"}
1274
+
1275
+
1276
+ def analyze_ocr(book: Book) -> dict:
1277
+ """Score OCR/conversion damage over the (pre-read) spine, skipping nav.
1278
+
1279
+ Primary signal: mid-sentence paragraph splits -- a prose block ends without
1280
+ terminal punctuation (last char a lowercase letter or comma) and the next
1281
+ prose block starts lowercase. Splits are only paired within one spine doc;
1282
+ chapter-file boundaries never produce a pair, and a pair interrupted by a
1283
+ rendered image (inline formula, card diagram) is cleared."""
1284
+ paras = 0
1285
+ splits = 0
1286
+ func_splits = 0
1287
+ examples: list[dict] = []
1288
+ for doc in book.spine:
1289
+ if book.nav and doc == book.nav:
1290
+ continue
1291
+ parser = _Blocks()
1292
+ try:
1293
+ parser.feed(book.docs.get(doc, ""))
1294
+ except Exception:
1295
+ continue
1296
+ li = sum(1 for t, _ in parser.blocks if t == "li")
1297
+ # a doc that is mostly <li> is a TOC / page-list, not body text
1298
+ if parser.blocks and li / len(parser.blocks) > 0.5:
1299
+ continue
1300
+ # a back-of-book index rendered as <p> blocks is not body text either
1301
+ if (
1302
+ sum(1 for _, t in parser.blocks if _OCR_INDEX_RE.search(t))
1303
+ >= _OCR_INDEX_MIN_BLOCKS
1304
+ ):
1305
+ continue
1306
+ paras += sum(1 for t, _ in parser.blocks if t in ("p", "div"))
1307
+ # pairs must be adjacent in reading order: an intervening heading /
1308
+ # scene-break block means a boundary, not a mid-sentence split
1309
+ for i in range(len(parser.blocks) - 1):
1310
+ ptag, prev = parser.blocks[i]
1311
+ ntag, nxt = parser.blocks[i + 1]
1312
+ if ptag not in ("p", "div") or ntag not in ("p", "div"):
1313
+ continue
1314
+ if parser.img_before[i + 1]:
1315
+ continue
1316
+ # a display quotation starts (and its resumption ends) mid-sentence
1317
+ # by design; never pair into or out of a <blockquote>
1318
+ if parser.in_quote[i] or parser.in_quote[i + 1]:
1319
+ continue
1320
+ # a dangling short unterminated fragment is a sign-off, not a
1321
+ # sentence remnant; a mostly-non-alphabetic fragment is display math
1322
+ if len(nxt) < _OCR_TAIL_MIN and nxt[-1] not in _OCR_TERMINALS:
1323
+ continue
1324
+ if min(_alpha_density(prev), _alpha_density(nxt)) < _OCR_ALPHA_MIN:
1325
+ continue
1326
+ unfinished = prev[-1].islower() or prev[-1] == ","
1327
+ lower_start = nxt[0].islower()
1328
+ substantial = len(prev) > _OCR_PROSE_MIN or len(nxt) > _OCR_PROSE_MIN
1329
+ if unfinished and lower_start and substantial:
1330
+ words = prev.rstrip(",").split()
1331
+ last = words[-1].lower() if words else ""
1332
+ # "Lordat noticed that" / lowercase block is the block-
1333
+ # quotation idiom of academic prose, not damage: an attribution
1334
+ # ending on "that" introduces a quote set as its own block,
1335
+ # which rightly starts lowercase
1336
+ if last == "that":
1337
+ continue
1338
+ splits += 1
1339
+ if last in OCR_FUNC_WORDS:
1340
+ func_splits += 1
1341
+ if len(examples) < 8:
1342
+ examples.append({"prev": prev[-60:], "next": nxt[:60]})
1343
+
1344
+ # Side signals over the visible text (dictionary-free by design).
1345
+ text = " ".join(book.visible_texts())
1346
+ glued: list[str] = []
1347
+ for w in set(_CAMEL_RE.findall(text)):
1348
+ hyphenated = _HUMP_RE.sub("-", w)
1349
+ if hyphenated in text:
1350
+ glued.append(f"{w}~{hyphenated}")
1351
+ return {
1352
+ "paras": paras,
1353
+ "splits": splits,
1354
+ "split_rate": splits / paras if paras else 0.0,
1355
+ "func_frac": func_splits / splits if splits else 0.0,
1356
+ "en_dash_words": len(EN_DASH_WORD_RE.findall(text)),
1357
+ "doubled_quotes": len(DOUBLED_QUOTE_RE.findall(text)),
1358
+ "glued": sorted(glued)[:8],
1359
+ "examples": examples,
1360
+ }
1361
+
1362
+
1363
+ def is_ocr_damaged(r: dict) -> bool:
1364
+ return (
1365
+ r["paras"] >= OCR_MIN_PARAS
1366
+ and r["splits"] >= OCR_MIN_SPLITS
1367
+ and r["split_rate"] >= OCR_FLAG_RATE
1368
+ and r["func_frac"] >= OCR_FUNC_MIN
1369
+ )
1370
+
1371
+
1372
+ def _ocr_detail(r: dict) -> str:
1373
+ bits = [
1374
+ f"{r['splits']} mid-sentence splits / {r['paras']} paragraphs "
1375
+ f"({r['split_rate'] * 100:.1f}%, {r['func_frac'] * 100:.0f}% on function words)"
1376
+ ]
1377
+ if r["en_dash_words"]:
1378
+ bits.append(f"en-dash-in-word x{r['en_dash_words']}")
1379
+ if r["doubled_quotes"]:
1380
+ bits.append(f"doubled quotes x{r['doubled_quotes']}")
1381
+ if r["glued"]:
1382
+ bits.append("glued nouns: " + ", ".join(r["glued"][:3]))
1383
+ return ", ".join(bits)
1384
+
1385
+
1386
+ def scan_ocr(path: Path) -> dict:
1387
+ return analyze_ocr(load_book(path))
1388
+
1389
+
1390
+ # ----------------------------------------------------------------------------
1391
+ # Per-analyzer reporting (library mode)
1392
+ # ----------------------------------------------------------------------------
1393
+
1394
+
1395
+ def _content_sections(nonlatin_hits, latin_foreign, signature_hits) -> int:
1396
+ """Print the content sections; return 1 if any unexpected hit or signature."""
1397
+
1398
+ def show(label: str, hits: list[tuple], color: str) -> int:
1399
+ unexpected = [h for h in hits if not h[3]]
1400
+ expected = [h for h in hits if h[3]]
1401
+ if hits:
1402
+ print(f"{color}{BOLD}{label} ({len(hits)}){RESET}")
1403
+ for book_id, title, tag, _exp, detail in sorted(unexpected):
1404
+ print(f" {RED}#{book_id}{RESET} [{tag}] {title}")
1405
+ print(f" {detail}")
1406
+ for book_id, title, tag, _exp, detail in sorted(expected):
1407
+ print(f" #{book_id} [{tag}] {title} {GREEN}(expected-foreign){RESET}")
1408
+ print(f" {detail}")
1409
+
1410
+ return len(unexpected)
1411
+
1412
+ unexpected = 0
1413
+ unexpected += show("NON-LATIN SCRIPT", nonlatin_hits, RED)
1414
+ unexpected += show("LATIN-SCRIPT FOREIGN (stopword vote)", latin_foreign, RED)
1415
+ # An injected piracy/ad notice is a defect no matter what language the book
1416
+ # is declared in, so the expected-foreign flag never applies here: without
1417
+ # this, a signature hit on a declared-foreign book printed "(expected-
1418
+ # foreign)" and "0 file(s) need review" while still failing the run.
1419
+ signature_hits = [(b, t, g, False, d) for b, t, g, _e, d in signature_hits]
1420
+ unexpected += show("INJECTION SIGNATURE", signature_hits, YELLOW)
1421
+ if unexpected == 0 and not signature_hits:
1422
+ print(f"{GREEN}{BOLD}content CLEAN{RESET}: no unexpected foreign content.")
1423
+ return 0
1424
+ print(
1425
+ f"{RED}{BOLD}content FOUND{RESET}: {unexpected} file(s) need review "
1426
+ f"(replace wrong-language editions with English copies)."
1427
+ )
1428
+ return 1
1429
+
1430
+
1431
+ def _pagenum_sections(found) -> int:
1432
+ if found:
1433
+ print(f"{RED}{BOLD}BAKED-IN PAGE NUMBERS ({len(found)}){RESET}")
1434
+ for book_id, title, tag, r in sorted(found, key=lambda x: -x[3]["n_hits"]):
1435
+ mark = f" {YELLOW}[watermark]{RESET}" if r["watermark"] else ""
1436
+ print(
1437
+ f" {RED}#{book_id}{RESET} [{tag}] {title}{mark}\n"
1438
+ f" {r['n_hits']} baked numbers, {r['span'] * 100:.0f}% of book, run {r['run']}"
1439
+ )
1440
+ for ex in r["examples"][:3]:
1441
+ print(f" ...{ex['prev']} {BOLD}{ex['v']}{RESET} {ex['next']}...")
1442
+ print()
1443
+ print(
1444
+ f"{RED}{BOLD}pagenumbers FOUND{RESET}: {len(found)} file(s) need review "
1445
+ f"(re-source and replace bad conversions)."
1446
+ )
1447
+ return 1
1448
+ print(f"{GREEN}{BOLD}pagenumbers CLEAN{RESET}: no baked-in page numbers found.")
1449
+ return 0
1450
+
1451
+
1452
+ def _empty_sections(empty, partial, thin) -> int:
1453
+ if empty:
1454
+ print(f"{RED}{BOLD}EMPTY / NO TEXT ({len(empty)}){RESET}")
1455
+ for book_id, title, tag, r in sorted(empty, key=lambda x: x[3]["chars"]):
1456
+ print(f" {RED}#{book_id}{RESET} [{tag}] {title}\n {_empty_detail(r)}")
1457
+ print()
1458
+ if partial:
1459
+ print(f"{RED}{BOLD}PARTIAL / PLACEHOLDER EXPORT ({len(partial)}){RESET}")
1460
+ for book_id, title, tag, r in sorted(partial, key=lambda x: x[3]["chars"]):
1461
+ print(f" {RED}#{book_id}{RESET} [{tag}] {title}\n {_empty_detail(r)}")
1462
+ print()
1463
+ if thin:
1464
+ print(
1465
+ f"{YELLOW}{BOLD}THIN (review; may be legitimately short) ({len(thin)}){RESET}"
1466
+ )
1467
+ for book_id, title, tag, r in sorted(thin, key=lambda x: x[3]["chars"]):
1468
+ print(
1469
+ f" {YELLOW}#{book_id}{RESET} [{tag}] {title}\n {_empty_detail(r)}"
1470
+ )
1471
+ print()
1472
+ found = len(empty) + len(partial)
1473
+ if found:
1474
+ print(f"{RED}{BOLD}emptytext FOUND{RESET}: {found} file(s) need re-sourcing.")
1475
+ return 1
1476
+ suffix = f" ({len(thin)} thin, advisory)" if thin else ""
1477
+ print(f"{GREEN}{BOLD}emptytext CLEAN{RESET}: every EPUB has body text{suffix}.")
1478
+ return 0
1479
+
1480
+
1481
+ def _ocr_sections(found) -> int:
1482
+ if found:
1483
+ print(f"{RED}{BOLD}OCR-DAMAGED PROSE ({len(found)}){RESET}")
1484
+ for book_id, title, tag, r in sorted(found, key=lambda x: -x[3]["split_rate"]):
1485
+ print(f" {RED}#{book_id}{RESET} [{tag}] {title}\n {_ocr_detail(r)}")
1486
+ for ex in r["examples"][:3]:
1487
+ print(f" ...{ex['prev']} {BOLD}/{RESET} {ex['next']}...")
1488
+ print()
1489
+ print(
1490
+ f"{RED}{BOLD}ocr FOUND{RESET}: {len(found)} file(s) need review "
1491
+ f"(re-source and replace damaged conversions)."
1492
+ )
1493
+ return 1
1494
+ print(f"{GREEN}{BOLD}ocr CLEAN{RESET}: no OCR-damaged prose found.")
1495
+ return 0
1496
+
1497
+
1498
+ def _monolithic_sections(hits) -> int:
1499
+ if hits:
1500
+ print(f"{RED}{BOLD}MONOLITHIC DOCUMENTS ({len(hits)}){RESET}")
1501
+ for book_id, title, tag, r in sorted(
1502
+ hits, key=lambda x: -x[3]["max_doc_chars"]
1503
+ ):
1504
+ print(f" {RED}#{book_id}{RESET} [{tag}] {title}")
1505
+ print(
1506
+ f" max doc {r['max_doc_chars']:,} chars ({r['worst_doc']});"
1507
+ f" spine {r['spine_len']}"
1508
+ )
1509
+ print()
1510
+ print(
1511
+ f"{RED}{BOLD}monolithic FOUND{RESET}: {len(hits)} file(s) need review "
1512
+ f"(readers may refuse to render; re-source or split by hand)."
1513
+ )
1514
+ return 1
1515
+ print(f"{GREEN}{BOLD}monolithic CLEAN{RESET}: no oversized single document.")
1516
+ return 0
1517
+
1518
+
1519
+ def _corrupt_sections(hits) -> int:
1520
+ if hits:
1521
+ print(f"{RED}{BOLD}CORRUPT ARCHIVES ({len(hits)}){RESET}")
1522
+ for book_id, title, tag, n, first in sorted(hits):
1523
+ print(f" {RED}#{book_id}{RESET} [{tag}] {title}")
1524
+ print(f" corrupt:{n} (first: {first}) — damaged archive; re-source")
1525
+ print()
1526
+ print(
1527
+ f"{RED}{BOLD}archive FOUND{RESET}: {len(hits)} file(s) need re-sourcing "
1528
+ f"(a damaged archive is not a content decision)."
1529
+ )
1530
+ return 1
1531
+ print(f"{GREEN}{BOLD}archive CLEAN{RESET}: every entry fully readable.")
1532
+ return 0
1533
+
1534
+
1535
+ def _spine_sections(hits, advisory) -> int:
1536
+ if advisory:
1537
+ print(
1538
+ f"{YELLOW}{BOLD}TOC MANIFEST CONVENTION ({len(advisory)}; reported, not flagged){RESET}"
1539
+ )
1540
+ for book_id, title, tag, r in sorted(advisory):
1541
+ print(f" {YELLOW}#{book_id}{RESET} [{tag}] {title}")
1542
+ print(f" toc {r['refs']} refs, {r['absent']} absent ({r['class']})")
1543
+ print()
1544
+ if hits:
1545
+ print(f"{RED}{BOLD}SPINE FRAGMENTS ({len(hits)}){RESET}")
1546
+ for book_id, title, tag, r in sorted(hits):
1547
+ print(f" {RED}#{book_id}{RESET} [{tag}] {title}")
1548
+ print(f" toc {r['refs']} refs, {r['absent']} absent — span broken")
1549
+ print()
1550
+ print(
1551
+ f"{RED}{BOLD}spine FOUND{RESET}: {len(hits)} file(s) are fragments "
1552
+ f"of themselves; quarantine."
1553
+ )
1554
+ return 1
1555
+ if not advisory:
1556
+ print(f"{GREEN}{BOLD}spine CLEAN{RESET}: every ToC target present.")
1557
+ return 0
1558
+
1559
+
1560
+ # ----------------------------------------------------------------------------
1561
+ # Runner
1562
+ # ----------------------------------------------------------------------------
1563
+
1564
+ ALL: tuple[str, ...] = ("content", "pagenumbers", "emptytext", "ocr", "monolithic")
1565
+
1566
+
1567
+ def run_library(
1568
+ selected: list[str],
1569
+ min_chars: int,
1570
+ thin_chars: int,
1571
+ tag: str | None = None,
1572
+ max_doc_chars: int = DEFAULT_MAX_DOC_CHARS,
1573
+ ) -> int:
1574
+ # The scan loop below reuses `tag` for its per-book display column; keep
1575
+ # the --tag argument under a distinct name so it survives that shadowing.
1576
+ audit_tag = tag
1577
+ library_root = resolve_library_root()
1578
+ if library_root is None:
1579
+ print(
1580
+ "ERROR: no metadata.db next to this script or in the current "
1581
+ "directory. Run from the library directory."
1582
+ )
1583
+ return 2
1584
+ db_path = library_root / "metadata.db"
1585
+
1586
+ from cquarry.db import CalibreDB
1587
+
1588
+ try:
1589
+ db = CalibreDB(str(db_path))
1590
+ except Exception as e:
1591
+ print(f"ERROR: cannot open {db_path}: {e}")
1592
+ return 2
1593
+ try:
1594
+ cur = db.conn.cursor()
1595
+ booktags: dict[int, list[str]] = {}
1596
+ for bid, tname in cur.execute(
1597
+ "SELECT bt.book, t.name FROM books_tags_link bt JOIN tags t ON t.id = bt.tag"
1598
+ ):
1599
+ booktags.setdefault(bid, []).append(tname)
1600
+ declared: dict[int, set[str]] = {}
1601
+ for bid, lang in cur.execute(
1602
+ "SELECT bl.book, l.lang_code FROM books_languages_link bl "
1603
+ "JOIN languages l ON l.id = bl.lang_code"
1604
+ ):
1605
+ declared.setdefault(bid, set()).add(lang)
1606
+ rows = cur.execute(
1607
+ "SELECT b.id, b.title FROM books b "
1608
+ "JOIN data d ON d.book = b.id WHERE d.format = 'EPUB' ORDER BY b.id"
1609
+ ).fetchall()
1610
+ # Canonical path resolution (library root / books.path / name.epub)
1611
+ # comes from cquarry so the layout logic lives in exactly one place.
1612
+ epubs = [
1613
+ (
1614
+ bid,
1615
+ title,
1616
+ Path(db.get_format_path(bid, "EPUB", verify=False)),
1617
+ )
1618
+ for bid, title in rows
1619
+ ]
1620
+ finally:
1621
+ db.close()
1622
+
1623
+ nonlatin_hits: list[tuple] = []
1624
+ latin_foreign: list[tuple] = []
1625
+ signature_hits: list[tuple] = []
1626
+ pagenum_found: list[tuple] = []
1627
+ empty_hits: list[tuple] = []
1628
+ partial_hits: list[tuple] = []
1629
+ thin_hits: list[tuple] = []
1630
+ ocr_found: list[tuple] = []
1631
+ mono_hits: list[tuple] = []
1632
+ corrupt_hits: list[tuple] = []
1633
+ spine_hits: list[tuple] = []
1634
+ spine_advisory: list[tuple] = []
1635
+ errors: list[tuple] = []
1636
+ scanned = 0
1637
+
1638
+ for book_id, title, full in ui.tqdm(epubs, desc=ui.info("Scanning library")):
1639
+ tags = booktags.get(book_id, [])
1640
+ tag = tags[0] if tags else "?"
1641
+ try:
1642
+ book = load_book(full)
1643
+ except Exception as e:
1644
+ errors.append((book_id, title, tag, f"{type(e).__name__}: {e}"))
1645
+ continue
1646
+ scanned += 1
1647
+ corrupt_r = analyze_corrupt(book)
1648
+ if corrupt_r["n"]:
1649
+ corrupt_hits.append(
1650
+ (book_id, title, tag, corrupt_r["n"], corrupt_r["first"])
1651
+ )
1652
+ spine_r = spine_integrity(book)
1653
+ if spine_r["class"] == "fragment":
1654
+ spine_hits.append((book_id, title, tag, spine_r))
1655
+ elif spine_r["class"] != "ok":
1656
+ spine_advisory.append((book_id, title, tag, spine_r))
1657
+
1658
+ if "content" in selected:
1659
+ r = analyze_content(book)
1660
+ langs = declared.get(book_id, set())
1661
+ decl = ",".join(sorted(langs)) if langs else "?"
1662
+ expected = any(lang != "eng" for lang in langs) or any(
1663
+ t.startswith("NonFic.Language.") for t in tags
1664
+ )
1665
+ for category, detail in findings(r):
1666
+ if category == "NON-LATIN SCRIPT":
1667
+ nonlatin_hits.append(
1668
+ (book_id, title, tag, expected, f"{detail}; declared={decl}")
1669
+ )
1670
+ elif category == "LATIN-SCRIPT FOREIGN":
1671
+ latin_foreign.append(
1672
+ (book_id, title, tag, expected, f"{detail}; declared={decl}")
1673
+ )
1674
+ else:
1675
+ signature_hits.append((book_id, title, tag, expected, detail))
1676
+
1677
+ if "pagenumbers" in selected:
1678
+ r = analyze_pagenumbers(book)
1679
+ if is_defective(r):
1680
+ pagenum_found.append((book_id, title, tag, r))
1681
+
1682
+ if "emptytext" in selected and not corrupt_r["n"]:
1683
+ r = analyze_emptytext(book)
1684
+ verdict = classify(r, min_chars, thin_chars)
1685
+ if verdict == "EMPTY":
1686
+ empty_hits.append((book_id, title, tag, r))
1687
+ elif verdict == "PARTIAL":
1688
+ partial_hits.append((book_id, title, tag, r))
1689
+ elif verdict == "THIN":
1690
+ thin_hits.append((book_id, title, tag, r))
1691
+
1692
+ if "ocr" in selected:
1693
+ r = analyze_ocr(book)
1694
+ if is_ocr_damaged(r):
1695
+ ocr_found.append((book_id, title, tag, r))
1696
+
1697
+ if "monolithic" in selected:
1698
+ r = analyze_monolithic(book)
1699
+ if is_monolithic(r, max_doc_chars):
1700
+ mono_hits.append((book_id, title, tag, r))
1701
+
1702
+ print(f"Scanned {scanned} EPUBs in {library_root}\n")
1703
+ rc = 0
1704
+ multi = len(selected) > 1
1705
+ for key in ALL:
1706
+ if key not in selected:
1707
+ continue
1708
+ if multi:
1709
+ print(f"{BOLD}== {key} =={RESET}")
1710
+ if key == "content":
1711
+ rc |= _content_sections(nonlatin_hits, latin_foreign, signature_hits)
1712
+ elif key == "pagenumbers":
1713
+ rc |= _pagenum_sections(pagenum_found)
1714
+ elif key == "emptytext":
1715
+ rc |= _empty_sections(empty_hits, partial_hits, thin_hits)
1716
+ elif key == "archive":
1717
+ rc |= _corrupt_sections(corrupt_hits)
1718
+ elif key == "spine":
1719
+ rc |= _spine_sections(spine_hits, spine_advisory)
1720
+ elif key == "monolithic":
1721
+ rc |= _monolithic_sections(mono_hits)
1722
+ else:
1723
+ rc |= _ocr_sections(ocr_found)
1724
+ if multi:
1725
+ print()
1726
+
1727
+ if errors:
1728
+ print(f"{YELLOW}{BOLD}SCAN ERRORS ({len(errors)}){RESET}")
1729
+ for book_id, title, tag, msg in errors:
1730
+ print(f" #{book_id} [{tag}] {title}\n {msg}")
1731
+ print()
1732
+ rc |= 1
1733
+
1734
+ if audit_tag:
1735
+ rc |= _apply_audit_tag(
1736
+ library_root,
1737
+ audit_tag,
1738
+ {
1739
+ "content": [h[0] for h in nonlatin_hits]
1740
+ + [h[0] for h in latin_foreign]
1741
+ + [h[0] for h in signature_hits],
1742
+ "pagenumbers": [h[0] for h in pagenum_found],
1743
+ "emptytext": [
1744
+ h[0] for h in empty_hits + partial_hits
1745
+ ], # THIN is advisory and stays untagged
1746
+ "ocr": [h[0] for h in ocr_found],
1747
+ "monolithic": [h[0] for h in mono_hits],
1748
+ "archive": [h[0] for h in corrupt_hits],
1749
+ "spine": [h[0] for h in spine_hits],
1750
+ },
1751
+ )
1752
+ return rc
1753
+
1754
+
1755
+ def _apply_audit_tag(
1756
+ library_root: Path, tag: str, flagged: dict[str, list[int]]
1757
+ ) -> int:
1758
+ """Apply ``tag`` to every flagged book via cquarry's opt-in write path.
1759
+
1760
+ Only reached with the explicit --tag flag; the audit itself stays strictly
1761
+ read-only. Calibre should be closed so the write does not fight its lock.
1762
+ Returns 0 on success, 2 on setup failure.
1763
+ """
1764
+ from cquarry.write import WritableCalibreDB
1765
+
1766
+ ids = sorted({bid for bids in flagged.values() for bid in bids})
1767
+ if not ids:
1768
+ print("Nothing flagged; no tags applied.")
1769
+ return 0
1770
+ db_path = library_root / "metadata.db"
1771
+ try:
1772
+ with WritableCalibreDB(str(db_path)) as wdb:
1773
+ applied = 0
1774
+ for bid in ids:
1775
+ if wdb.add_tag(bid, tag):
1776
+ applied += 1
1777
+ except Exception as e:
1778
+ print(f"ERROR: tagging failed ({type(e).__name__}: {e}).")
1779
+ print("Is Calibre closed? The write path refuses to fight its lock.")
1780
+ return 2
1781
+ print(
1782
+ f"Tagged {applied} of {len(ids)} flagged books with [{tag}]"
1783
+ " (already-tagged books skipped)."
1784
+ )
1785
+ return 0
1786
+
1787
+
1788
+ def _content_dir(r: dict) -> tuple[bool, str, list[str]]:
1789
+ hits = findings(r)
1790
+ if not hits:
1791
+ return False, "OK", []
1792
+ return True, "REVIEW", [f"{cat}: {detail}" for cat, detail in hits]
1793
+
1794
+
1795
+ def _pagenum_dir(r: dict) -> tuple[bool, str, list[str]]:
1796
+ if not is_defective(r):
1797
+ return False, "OK", []
1798
+ mark = " [watermark]" if r["watermark"] else ""
1799
+ lines = [
1800
+ f"{r['n_hits']} baked numbers, {r['span'] * 100:.0f}% of book, run {r['run']}{mark}"
1801
+ ]
1802
+ lines += [
1803
+ f"...{ex['prev']} {ex['v']} {ex['next']}..." for ex in r["examples"][:2]
1804
+ ]
1805
+ return True, "REVIEW", lines
1806
+
1807
+
1808
+ def _empty_dir(r: dict, min_chars: int, thin_chars: int) -> tuple[bool, str, list[str]]:
1809
+ verdict = classify(r, min_chars, thin_chars)
1810
+ if verdict == "OK":
1811
+ return False, "OK", []
1812
+ if verdict == "CORRUPT":
1813
+ return True, "CORRUPT", [_empty_detail(r)]
1814
+ return verdict in ("EMPTY", "PARTIAL"), verdict, [_empty_detail(r)]
1815
+
1816
+
1817
+ def _monolithic_dir(r: dict, max_doc_chars: int) -> tuple[bool, str, list[str]]:
1818
+ if is_monolithic(r, max_doc_chars):
1819
+ return (
1820
+ True,
1821
+ "FLAG",
1822
+ [f"max doc {r['max_doc_chars']:,} chars ({r['worst_doc']})"],
1823
+ )
1824
+ return False, "OK", [f"max doc {r['max_doc_chars']:,} chars"]
1825
+
1826
+
1827
+ def _ocr_dir(r: dict) -> tuple[bool, str, list[str]]:
1828
+ if not is_ocr_damaged(r):
1829
+ return False, "OK", []
1830
+ lines = [_ocr_detail(r)]
1831
+ lines += [f"...{ex['prev']} / {ex['next']}..." for ex in r["examples"][:2]]
1832
+ return True, "REVIEW", lines
1833
+
1834
+
1835
+ def run_directory(
1836
+ directory: Path,
1837
+ selected: list[str],
1838
+ min_chars: int,
1839
+ thin_chars: int,
1840
+ max_doc_chars: int = DEFAULT_MAX_DOC_CHARS,
1841
+ ) -> int:
1842
+ if not directory.is_dir():
1843
+ print(f"ERROR: {directory} is not a directory.")
1844
+ return 2
1845
+ epubs = sorted(directory.rglob("*.epub"))
1846
+ if not epubs:
1847
+ print(f"No .epub files found under {directory}")
1848
+ return 2
1849
+
1850
+ print(f"Auditing {len(epubs)} EPUB(s) in {directory}\n")
1851
+ multi = len(selected) > 1
1852
+ problems = 0
1853
+ errors = 0
1854
+ for path in ui.tqdm(epubs, desc=ui.info("Scanning directory")):
1855
+ try:
1856
+ book = load_book(path)
1857
+ except Exception as e:
1858
+ ui.tqdm.write(
1859
+ f" {YELLOW}ERROR {RESET} {path.name}\n {type(e).__name__}: {e}"
1860
+ )
1861
+ errors += 1
1862
+ continue
1863
+
1864
+ verdicts = []
1865
+ corrupt_r = analyze_corrupt(book)
1866
+ spine_r = spine_integrity(book)
1867
+ for key in ALL:
1868
+ if key not in selected:
1869
+ continue
1870
+ if key == "emptytext" and corrupt_r["n"]:
1871
+ continue # the archive verdict owns this book's body-text story
1872
+ if key == "content":
1873
+ problem, status, lines = _content_dir(analyze_content(book))
1874
+ elif key == "pagenumbers":
1875
+ problem, status, lines = _pagenum_dir(analyze_pagenumbers(book))
1876
+ elif key == "emptytext":
1877
+ problem, status, lines = _empty_dir(
1878
+ analyze_emptytext(book), min_chars, thin_chars
1879
+ )
1880
+ elif key == "monolithic":
1881
+ problem, status, lines = _monolithic_dir(
1882
+ analyze_monolithic(book), max_doc_chars
1883
+ )
1884
+ else:
1885
+ problem, status, lines = _ocr_dir(analyze_ocr(book))
1886
+ if problem:
1887
+ problems += 1
1888
+ verdicts.append((key, problem, status, lines))
1889
+
1890
+ if corrupt_r["n"]:
1891
+ problem, status, lines = _corrupt_verdict(corrupt_r)
1892
+ problems += 1
1893
+ verdicts.append(("archive", problem, status, lines))
1894
+
1895
+ if spine_r["class"] != "ok":
1896
+ problem, status, lines = _spine_verdict(spine_r)
1897
+ if problem:
1898
+ problems += 1
1899
+ verdicts.append(("spine", problem, status, lines))
1900
+
1901
+ if multi:
1902
+ ui.tqdm.write(f" {path.name}")
1903
+ for key, problem, status, lines in verdicts:
1904
+ color = RED if problem else (YELLOW if status != "OK" else GREEN)
1905
+ ui.tqdm.write(f" {color}{status:<6}{RESET} {key}")
1906
+ for ln in lines:
1907
+ ui.tqdm.write(f" {ln}")
1908
+ else:
1909
+ _key, problem, status, lines = verdicts[0]
1910
+ color = RED if problem else (YELLOW if status != "OK" else GREEN)
1911
+ ui.tqdm.write(f" {color}{status:<6}{RESET} {path.name}")
1912
+ for ln in lines:
1913
+ ui.tqdm.write(f" {ln}")
1914
+ print()
1915
+
1916
+ if problems == 0 and errors == 0:
1917
+ print(f"{GREEN}{BOLD}CLEAN{RESET}: no problems in {len(epubs)} file(s).")
1918
+ return 0
1919
+ print(
1920
+ f"{RED}{BOLD}FOUND{RESET}: {problems} problem(s) need review, "
1921
+ f"{errors} scan error(s)."
1922
+ )
1923
+ return 1
1924
+
1925
+
1926
+ def run_single(
1927
+ book_id: int,
1928
+ selected: list[str],
1929
+ min_chars: int,
1930
+ thin_chars: int,
1931
+ tag: str | None = None,
1932
+ max_doc_chars: int = DEFAULT_MAX_DOC_CHARS,
1933
+ ) -> int:
1934
+ """Audit one library book by id — cquarry's single-entity fetch.
1935
+
1936
+ Uses :meth:`CalibreDB.get_book` so only that book's row is read instead
1937
+ of caching the entire library layout, then resolves the EPUB through
1938
+ cquarry's ``get_format_path``. The audit itself stays strictly read-only;
1939
+ ``--tag`` applies via cquarry's opt-in write path only when flagged.
1940
+ """
1941
+ library_root = resolve_library_root()
1942
+ if library_root is None:
1943
+ print(
1944
+ "ERROR: no metadata.db next to this script or in the current "
1945
+ "directory. Run from the library directory."
1946
+ )
1947
+ return 2
1948
+
1949
+ from cquarry.db import CalibreDB
1950
+
1951
+ try:
1952
+ db = CalibreDB(str(library_root / "metadata.db"))
1953
+ except Exception as e:
1954
+ print(f"ERROR: cannot open {library_root / 'metadata.db'}: {e}")
1955
+ return 2
1956
+ try:
1957
+ rec = db.get_book(book_id)
1958
+ if rec is None:
1959
+ print(f"ERROR: no book #{book_id} in {library_root}")
1960
+ return 2
1961
+ title = rec["title"]
1962
+ tags = rec["tags"]
1963
+ try:
1964
+ path = Path(db.get_format_path(book_id, "EPUB", verify=True))
1965
+ except (ValueError, FileNotFoundError) as e:
1966
+ print(f"ERROR: book #{book_id} ({title}): {e}")
1967
+ return 2
1968
+ finally:
1969
+ db.close()
1970
+
1971
+ try:
1972
+ book = load_book(path)
1973
+ except Exception as e:
1974
+ print(f"ERROR reading {path.name}: {type(e).__name__}: {e}")
1975
+ return 1
1976
+
1977
+ tag_display = tags[0] if tags else "?"
1978
+ print(f"Auditing #{book_id} [{tag_display}] {title}\n {path}\n")
1979
+ problems = 0
1980
+ multi = len(selected) > 1
1981
+ verdicts = []
1982
+ corrupt_r = analyze_corrupt(book)
1983
+ spine_r = spine_integrity(book)
1984
+ for key in ALL:
1985
+ if key not in selected:
1986
+ continue
1987
+ if key == "emptytext" and corrupt_r["n"]:
1988
+ # Corruption owns the body-text story for this book; reporting
1989
+ # EMPTY here would be the wrong disease.
1990
+ continue
1991
+ if key == "content":
1992
+ problem, status, lines = _content_dir(analyze_content(book))
1993
+ elif key == "pagenumbers":
1994
+ problem, status, lines = _pagenum_dir(analyze_pagenumbers(book))
1995
+ elif key == "emptytext":
1996
+ problem, status, lines = _empty_dir(
1997
+ analyze_emptytext(book), min_chars, thin_chars
1998
+ )
1999
+ elif key == "monolithic":
2000
+ problem, status, lines = _monolithic_dir(
2001
+ analyze_monolithic(book), max_doc_chars
2002
+ )
2003
+ else:
2004
+ problem, status, lines = _ocr_dir(analyze_ocr(book))
2005
+ if problem:
2006
+ problems += 1
2007
+ verdicts.append((key, problem, status, lines))
2008
+
2009
+ if corrupt_r["n"]:
2010
+ problem, status, lines = _corrupt_verdict(corrupt_r)
2011
+ problems += 1
2012
+ verdicts.append(("archive", problem, status, lines))
2013
+
2014
+ if spine_r["class"] != "ok":
2015
+ problem, status, lines = _spine_verdict(spine_r)
2016
+ if problem:
2017
+ problems += 1
2018
+ verdicts.append(("spine", problem, status, lines))
2019
+
2020
+ for key, problem, status, lines in verdicts:
2021
+ if multi:
2022
+ print(f" {key}")
2023
+ color = RED if problem else (YELLOW if status != "OK" else GREEN)
2024
+ prefix = " " if multi else " "
2025
+ print(f"{prefix}{color}{status:<6}{RESET} {key}")
2026
+ for ln in lines:
2027
+ print(f"{prefix} {ln}")
2028
+
2029
+ rc = 0
2030
+ if problems:
2031
+ rc = 1
2032
+ print(f"\n{RED}{BOLD}FOUND{RESET}: {problems} problem(s) need review.")
2033
+ else:
2034
+ print(f"\n{GREEN}{BOLD}CLEAN{RESET}: book #{book_id} passed the audit.")
2035
+ if tag and problems:
2036
+ rc |= _apply_audit_tag(
2037
+ library_root,
2038
+ tag,
2039
+ {key: [book_id] for key, problem, _s, _l in verdicts if problem},
2040
+ )
2041
+ return rc
2042
+
2043
+
2044
+ def main() -> int:
2045
+ parser = argparse.ArgumentParser(
2046
+ description="Audit EPUB body text for non-English content, baked-in page "
2047
+ "numbers, empty stubs, or OCR-damaged prose."
2048
+ )
2049
+ parser.add_argument(
2050
+ "mode",
2051
+ choices=("content", "pagenumbers", "emptytext", "ocr", "monolithic", "all"),
2052
+ help="which audit to run ('all' runs the analyzers in one decompression pass)",
2053
+ )
2054
+ parser.add_argument(
2055
+ "directory",
2056
+ nargs="?",
2057
+ help="vet loose .epub files under this directory instead of the library",
2058
+ )
2059
+ parser.add_argument(
2060
+ "--min-chars",
2061
+ type=int,
2062
+ default=DEFAULT_MIN_CHARS,
2063
+ help=f"emptytext EMPTY threshold (default {DEFAULT_MIN_CHARS})",
2064
+ )
2065
+ parser.add_argument(
2066
+ "--thin-chars",
2067
+ type=int,
2068
+ default=DEFAULT_THIN_CHARS,
2069
+ help=f"emptytext THIN advisory threshold (default {DEFAULT_THIN_CHARS})",
2070
+ )
2071
+ parser.add_argument(
2072
+ "--max-doc-chars",
2073
+ type=int,
2074
+ default=DEFAULT_MAX_DOC_CHARS,
2075
+ help=f"monolithic FLAG threshold (default {DEFAULT_MAX_DOC_CHARS})",
2076
+ )
2077
+ parser.add_argument(
2078
+ "--id",
2079
+ metavar="BOOK_IDS",
2080
+ default=None,
2081
+ help="audit library book(s) by Calibre id — one id or a comma-separated "
2082
+ "list (fetched via cquarry's single-entity get_book; cannot be "
2083
+ "combined with a directory)",
2084
+ )
2085
+ parser.add_argument(
2086
+ "--tag",
2087
+ metavar="TAG",
2088
+ default=None,
2089
+ help="library mode only: tag every flagged book via cquarry's opt-in "
2090
+ "write path (Calibre must be closed; audit itself stays read-only)",
2091
+ )
2092
+ args = parser.parse_args()
2093
+ ui.print_header("bindery audit - Execution")
2094
+ selected = list(ALL) if args.mode == "all" else [args.mode]
2095
+ if args.id is not None:
2096
+ if args.directory:
2097
+ print("ERROR: --id audits a library book; drop the directory argument.")
2098
+ return 2
2099
+ rc = 0
2100
+ for raw in str(args.id).split(","):
2101
+ raw = raw.strip()
2102
+ if not raw:
2103
+ continue
2104
+ try:
2105
+ bid = int(raw)
2106
+ except ValueError:
2107
+ print(f"ERROR: --id {raw!r} is not a book id.", file=sys.stderr)
2108
+ rc |= 2
2109
+ continue
2110
+ rc |= run_single(
2111
+ bid,
2112
+ selected,
2113
+ args.min_chars,
2114
+ args.thin_chars,
2115
+ tag=args.tag,
2116
+ max_doc_chars=args.max_doc_chars,
2117
+ )
2118
+ return rc
2119
+ if args.directory:
2120
+ return run_directory(
2121
+ Path(args.directory).expanduser(),
2122
+ selected,
2123
+ args.min_chars,
2124
+ args.thin_chars,
2125
+ max_doc_chars=args.max_doc_chars,
2126
+ )
2127
+ return run_library(
2128
+ selected,
2129
+ args.min_chars,
2130
+ args.thin_chars,
2131
+ tag=args.tag,
2132
+ max_doc_chars=args.max_doc_chars,
2133
+ )
2134
+
2135
+
2136
+ if __name__ == "__main__":
2137
+ sys.exit(main())