bindery-cli 0.35.0__tar.gz → 0.36.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/CLAUDE.md +3 -3
  2. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/PKG-INFO +10 -5
  3. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/README.md +8 -3
  4. bindery_cli-0.36.0/REPORT-12-Sept.md +119 -0
  5. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/patchnotes.md +52 -0
  6. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/pyproject.toml +2 -2
  7. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/roadmap.md +100 -3
  8. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/spec.md +16 -4
  9. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/__init__.py +1 -1
  10. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/audit.py +183 -5
  11. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/cli.py +13 -4
  12. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_audit.py +283 -0
  13. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/uv.lock +4 -4
  14. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/.github/workflows/ci.yml +0 -0
  15. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/.github/workflows/publish.yml +0 -0
  16. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/.gitignore +0 -0
  17. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/AGENTS.md +0 -0
  18. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/LICENSE +0 -0
  19. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/docs/screenshots/library-sweep.png +0 -0
  20. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/logo.svg +0 -0
  21. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/run_tests.sh +0 -0
  22. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/scripts/FastSweep.java +0 -0
  23. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/scripts/fast_sweep.py +0 -0
  24. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/__main__.py +0 -0
  25. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/epub.py +0 -0
  26. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/library.py +0 -0
  27. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/pagination.py +0 -0
  28. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/reserialize.py +0 -0
  29. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/transforms.py +0 -0
  30. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/validate.py +0 -0
  31. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/watermark.py +0 -0
  32. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/testing_facility/books_to_fix/.gitkeep +0 -0
  33. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/testing_facility/top500candidates/REPORT.md +0 -0
  34. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_cli.py +0 -0
  35. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_epub.py +0 -0
  36. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_library.py +0 -0
  37. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_pagination.py +0 -0
  38. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_reserialize.py +0 -0
  39. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_transforms.py +0 -0
  40. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_validate.py +0 -0
  41. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_version.py +0 -0
  42. {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_watermark.py +0 -0
@@ -77,12 +77,12 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
77
77
  - `src/bindery/pagination.py`: the opt-in lossy page-number strip (runhead detection, page-layer decision, block-centric removal/merge, safety nets).
78
78
  - `src/bindery/watermark.py`: the opt-in lossy watermark strip (anchored and anchorless signature removal).
79
79
  - `src/bindery/reserialize.py`: structural repair via `html5lib`.
80
- - `src/bindery/audit.py`: read-only body-text audits (`content`, `pagenumbers`, `emptytext`, `ocr`, and `monolithic` since v0.21.0, `--max-doc-chars N`) behind the `audit` subcommand (v0.15.0). Since v0.18.0 library mode resolves EPUB paths through `cquarry.get_format_path()` and can apply a tag to flagged books via `cquarry.write.WritableCalibreDB` (only with explicit `--tag`; the only sanctioned write path). Since v0.19.0 `audit --id BOOK_ID` audits a single library book through cquarry's single-entity `get_book()` fetch (no library-wide cache; supports `--tag`; incompatible with the directory argument; comma lists since v0.23.0). Since v0.22.0 every archive entry is fully read (CRC + decompression) before analysis — a damaged entry reports its own CORRUPT verdict instead of feeding emptytext — and manifest/NCX references to absent files classify as `convention` (bloated ToC, consecutive span) or `fragment` (broken span). Hazard: the scan loop reuses `tag` as its per-book display column, so the `--tag` argument is captured as `audit_tag` at function entry — do not collapse them again. Since v0.29.0 `audit --json FILE` writes per-file analyzer verdicts in the `library --json` shape (directory, library, and single-book modes; with `--id`, exactly one id; emptytext omitted from a record when the archive verdict owns the body-text story).
80
+ - `src/bindery/audit.py`: read-only body-text audits (`content`, `pagenumbers`, `emptytext`, `ocr`, `monolithic` since v0.21.0 with `--max-doc-chars N`, and `completeness` since v0.36.0) behind the `audit` subcommand (v0.15.0). Since v0.18.0 library mode resolves EPUB paths through `cquarry.get_format_path()` and can apply a tag to flagged books via `cquarry.write.WritableCalibreDB` (only with explicit `--tag`; the only sanctioned write path). Since v0.19.0 `audit --id BOOK_ID` audits a single library book through cquarry's single-entity `get_book()` fetch (no library-wide cache; supports `--tag`; incompatible with the directory argument; comma lists since v0.23.0). Since v0.22.0 every archive entry is fully read (CRC + decompression) before analysis — a damaged entry reports its own CORRUPT verdict instead of feeding emptytext — and manifest/NCX references to absent files classify as `convention` (bloated ToC, consecutive span) or `fragment` (broken span). Hazard: the scan loop reuses `tag` as its per-book display column, so the `--tag` argument is captured as `audit_tag` at function entry — do not collapse them again. Since v0.29.0 `audit --json FILE` writes per-file analyzer verdicts in the `library --json` shape (directory, library, and single-book modes; with `--id`, exactly one id; emptytext omitted from a record when the archive verdict owns the body-text story). Since v0.36.0 the completeness analyzer (Phase 15) reports the phase-1 spot-check per book: spine/prose-doc counts (prose = >= 400 visible chars), first/middle/last prose-doc opening/closing excerpts, trailing-ToC classification (`_trailing_toc`: short link lines dominate, no paragraph-length blocks), and the unreadable fraction. It is ADVISORY by contract (problem always False; ADVISORY on a trailing ToC or unreadable fraction >= 10%), never moves the exit code, and is skipped like emptytext when the archive verdict owns the book.
81
81
  - `src/bindery/epub.py`: archive rewrite, NCX uid sync, RepairReport, mismatch detection, and the opt-in structural-repair plumbing (including the CSS precondition scan).
82
82
  - `src/bindery/validate.py`: epubcheck wrapper, the `gate` (improvement) and `no_worse` (no-regression, for the lossy strips) acceptance bars.
83
- - `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching) — the `(id)` directory-name regex is only the no-catalog fallback. Since v0.24.0 `install_format()` places the repaired file atomically and updates the `data` row through `cquarry.write.WritableCalibreDB` (since v0.31.0 via `set_format`, cquarry 1.17's sanctioned remove+add in one transaction); the external `calibredb` CLI is gone. Since v0.32.0 a guessed id drives a row update only when metadata.db corroborates it (`_verify_guess`: books row exists, file in the book's own directory, catalogued `data.name` when a row exists); anything else saves in place and leaves the catalog untouched — never reintroduce directory-name guessing as the primary source: renamed/mismatched directories would replace the wrong book.
83
+ - `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching); the `(id)` directory-name regex is only the no-catalog fallback. Since v0.24.0 `install_format()` places the repaired file atomically and updates the `data` row through `cquarry.write.WritableCalibreDB` (since v0.31.0 via `set_format`, cquarry 1.17's sanctioned remove+add in one transaction); the external `calibredb` CLI is gone. Since v0.32.0 a guessed id drives a row update only when metadata.db corroborates it (`_verify_guess`: books row exists, file in the book's own directory, catalogued `data.name` when a row exists); anything else saves in place and leaves the catalog untouched — never reintroduce directory-name guessing as the primary source: renamed/mismatched directories would replace the wrong book.
84
84
  - `src/bindery/cli.py`: `repair` and `library` subcommands, including `--all` and `--install-to-calibre`; since v0.29.0 also the `run` verbs: `phase1` (pre-import vetting over loose files: the audit battery composed with one gated repair sweep; read-only until `--apply-lossy`, which IS the lossy-strip consent) and `phase3` (the scoped post-import apply step: `library --id --sweep --only all --apply --all --install-to-calibre` plus a pre/post summary; mechanically refuses unscoped sweeps with exit 2). Both take `--non-interactive` and surface open questions as `decisions_needed` in their JSON, never prompts. They drive the shipped `run_library` through the real parser, so no flag can drift between the verb and the subcommand it wraps. `library --workers N` (v0.30.0) parallelizes only the sweep's candidate pass, in windows of N consumed in order so `--limit` stays lazy; the repair phase is never parallel (shared workdir, atomic-replacement contract).
85
- - `tests/`: transforms, end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic, archive integrity, spine classification), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, and the version pin.
85
+ - `tests/`: transforms, end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic/completeness, archive integrity, spine classification), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, and the version pin.
86
86
 
87
87
  ## Conventions
88
88
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: bindery-cli
3
- Version: 0.35.0
3
+ Version: 0.36.0
4
4
  Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
5
5
  Author: vrnvctss
6
6
  License: MIT
@@ -13,7 +13,7 @@ Classifier: License :: OSI Approved :: MIT License
13
13
  Classifier: Programming Language :: Python :: 3.14
14
14
  Classifier: Topic :: Utilities
15
15
  Requires-Python: >=3.14
16
- Requires-Dist: cquarry>=1.17.0
16
+ Requires-Dist: cquarry>=1.19.0
17
17
  Requires-Dist: tqdm
18
18
  Requires-Dist: vir-tui>=2.2.0
19
19
  Provides-Extra: reserialize
@@ -57,8 +57,8 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
57
57
  - **`--unwrap-block-in-inline`**: safely unwraps `<span>` tags that illegally contain a block-level element (e.g. `<div>` or `<p>`), leaving the block element intact.
58
58
  - **`--strip-invalid-value`**: systematically strips invalid `value="..."` attributes from elements like `<div>`, `<span>`, `<p>`, etc.
59
59
  - **`--fix-page-map`**: normalizes legacy page-map markup — drops the non-standard `page-map="..."` attribute from the OPF `<spine>` and adds `class="pages"` to classless NCX `<pageList>` elements (epubcheck rejects both on older HarperCollins / Anna's Archive conversions).
60
- - **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package — `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive). The edit is anchored to real start tags, so prose that merely mentions `epub:type="chapter"` is preserved.
61
- - **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents — `figure`/`section` to `div`, `figcaption` to `p` — keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed. Both EPUB2-targeted fixes are gated on the package version declared in the OPF: on an EPUB 3 book they do nothing, because their target defects only exist below EPUB 3.
60
+ - **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package: `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive). The edit is anchored to real start tags, so prose that merely mentions `epub:type="chapter"` is preserved.
61
+ - **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents (`figure`/`section` to `div`, `figcaption` to `p`), keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed. Both EPUB2-targeted fixes are gated on the package version declared in the OPF: on an EPUB 3 book they do nothing, because their target defects only exist below EPUB 3.
62
62
  - **`--unwrap-illegal-tags`**: strips completely invalid or deprecated HTML tags that break EPUB3 validation (`<st>`, `<sentence>`, `<o>`, `<w>`, `<pagebreak>`) while retaining their inner text. Any of those names styled as an *element selector* by an EPUB stylesheet (`.css` entries and inline `<style>` blocks alike; class/id selectors like `.st`/`#w` don't count) is protected for the whole book, guaranteeing format preservation.
63
63
  - **`--prune-missing-resources`**: removes references to files the archive does not contain (RSC-007/PKG-010): dead `<link>` elements, anchors' `href` to absent files (anchor text preserved), absent `<img>` sources (replaced by their alt text when they carry one), and orphaned non-spine OPF manifest items. Spine documents are never pruned: a missing spine document is reported as a damaged fragment, not silently dropped.
64
64
  - **`--strip-broken-anchors`**: strips href attributes that cannot resolve, keeping the anchor text byte-for-byte: a `#fragment` the target document does not define (RSC-020/RSC-012; NCX navTargets fall back to the document target, so chapter navigation survives) and unresolvable URI schemes (`kindle:`, `file:`).
@@ -126,10 +126,15 @@ bindery audit ocr ~/docs/Calibre\ Library
126
126
  # Flag books where a single content document is too large for readers to render
127
127
  bindery audit monolithic ~/docs/Calibre\ Library
128
128
 
129
+ # Spot-check completeness: prose coverage, first/middle/last excerpts, trailing ToC
130
+ bindery audit completeness ~/docs/Calibre\ Library
131
+
129
132
  # Run all audits and generate a comprehensive CSV
130
133
  cd ~/docs/Calibre\ Library && bindery audit all
131
134
  ```
132
135
 
136
+ **Completeness spot-check:** the `completeness` audit answers the pre-import question "does this book reach its end?" It reports, per book: the prose-document count against the spine (docs with 400+ visible characters), the opening and closing text of the first/middle/last prose doc, a trailing-ToC classification of the final spine doc (a link-dense chapter list after the real back matter), and the fraction of unreadable docs. A trailing ToC is reported as an advisory, never a failure: the closing excerpt always comes from the real back matter, so a human can confirm the book ends where the author ended it.
137
+
133
138
  Audits can also be run on a directory of loose `.epub` files by passing the path as the second argument:
134
139
  ```sh
135
140
  bindery audit pagenumbers /path/to/loose/epubs
@@ -204,7 +209,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
204
209
  - `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
205
210
  - `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
206
211
  - `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
207
- - `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied` — Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning. A directory-name-guessed id is trusted only when `metadata.db` corroborates it (the book exists and the file is that book's catalogued EPUB); an uncatalogued stray file inside a book directory is saved in place and the catalog row is left untouched.
212
+ - `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied`; Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning. A directory-name-guessed id is trusted only when `metadata.db` corroborates it (the book exists and the file is that book's catalogued EPUB); an uncatalogued stray file inside a book directory is saved in place and the catalog row is left untouched.
208
213
  - `--all` automatically turns on all opt-in non-fatal fixes and lossy strips (pagination, watermarks, bad attributes, unknown entities, image alt tags, etc.) in a single run.
209
214
  - Only the `.epub` is replaced. `metadata.opf`, `cover.jpg`, and `metadata.db` are left for Calibre's Quality Check sync to reconcile.
210
215
  - A per-book progress line goes to stderr (stdout stays a clean report); `--quiet` suppresses it. A corrupt or unreadable book is reported and skipped, never aborting the sweep.
@@ -35,8 +35,8 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
35
35
  - **`--unwrap-block-in-inline`**: safely unwraps `<span>` tags that illegally contain a block-level element (e.g. `<div>` or `<p>`), leaving the block element intact.
36
36
  - **`--strip-invalid-value`**: systematically strips invalid `value="..."` attributes from elements like `<div>`, `<span>`, `<p>`, etc.
37
37
  - **`--fix-page-map`**: normalizes legacy page-map markup — drops the non-standard `page-map="..."` attribute from the OPF `<spine>` and adds `class="pages"` to classless NCX `<pageList>` elements (epubcheck rejects both on older HarperCollins / Anna's Archive conversions).
38
- - **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package — `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive). The edit is anchored to real start tags, so prose that merely mentions `epub:type="chapter"` is preserved.
39
- - **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents — `figure`/`section` to `div`, `figcaption` to `p` — keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed. Both EPUB2-targeted fixes are gated on the package version declared in the OPF: on an EPUB 3 book they do nothing, because their target defects only exist below EPUB 3.
38
+ - **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package: `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive). The edit is anchored to real start tags, so prose that merely mentions `epub:type="chapter"` is preserved.
39
+ - **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents (`figure`/`section` to `div`, `figcaption` to `p`), keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed. Both EPUB2-targeted fixes are gated on the package version declared in the OPF: on an EPUB 3 book they do nothing, because their target defects only exist below EPUB 3.
40
40
  - **`--unwrap-illegal-tags`**: strips completely invalid or deprecated HTML tags that break EPUB3 validation (`<st>`, `<sentence>`, `<o>`, `<w>`, `<pagebreak>`) while retaining their inner text. Any of those names styled as an *element selector* by an EPUB stylesheet (`.css` entries and inline `<style>` blocks alike; class/id selectors like `.st`/`#w` don't count) is protected for the whole book, guaranteeing format preservation.
41
41
  - **`--prune-missing-resources`**: removes references to files the archive does not contain (RSC-007/PKG-010): dead `<link>` elements, anchors' `href` to absent files (anchor text preserved), absent `<img>` sources (replaced by their alt text when they carry one), and orphaned non-spine OPF manifest items. Spine documents are never pruned: a missing spine document is reported as a damaged fragment, not silently dropped.
42
42
  - **`--strip-broken-anchors`**: strips href attributes that cannot resolve, keeping the anchor text byte-for-byte: a `#fragment` the target document does not define (RSC-020/RSC-012; NCX navTargets fall back to the document target, so chapter navigation survives) and unresolvable URI schemes (`kindle:`, `file:`).
@@ -104,10 +104,15 @@ bindery audit ocr ~/docs/Calibre\ Library
104
104
  # Flag books where a single content document is too large for readers to render
105
105
  bindery audit monolithic ~/docs/Calibre\ Library
106
106
 
107
+ # Spot-check completeness: prose coverage, first/middle/last excerpts, trailing ToC
108
+ bindery audit completeness ~/docs/Calibre\ Library
109
+
107
110
  # Run all audits and generate a comprehensive CSV
108
111
  cd ~/docs/Calibre\ Library && bindery audit all
109
112
  ```
110
113
 
114
+ **Completeness spot-check:** the `completeness` audit answers the pre-import question "does this book reach its end?" It reports, per book: the prose-document count against the spine (docs with 400+ visible characters), the opening and closing text of the first/middle/last prose doc, a trailing-ToC classification of the final spine doc (a link-dense chapter list after the real back matter), and the fraction of unreadable docs. A trailing ToC is reported as an advisory, never a failure: the closing excerpt always comes from the real back matter, so a human can confirm the book ends where the author ended it.
115
+
111
116
  Audits can also be run on a directory of loose `.epub` files by passing the path as the second argument:
112
117
  ```sh
113
118
  bindery audit pagenumbers /path/to/loose/epubs
@@ -182,7 +187,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
182
187
  - `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
183
188
  - `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
184
189
  - `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
185
- - `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied` — Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning. A directory-name-guessed id is trusted only when `metadata.db` corroborates it (the book exists and the file is that book's catalogued EPUB); an uncatalogued stray file inside a book directory is saved in place and the catalog row is left untouched.
190
+ - `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied`; Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning. A directory-name-guessed id is trusted only when `metadata.db` corroborates it (the book exists and the file is that book's catalogued EPUB); an uncatalogued stray file inside a book directory is saved in place and the catalog row is left untouched.
186
191
  - `--all` automatically turns on all opt-in non-fatal fixes and lossy strips (pagination, watermarks, bad attributes, unknown entities, image alt tags, etc.) in a single run.
187
192
  - Only the `.epub` is replaced. `metadata.opf`, `cover.jpg`, and `metadata.db` are left for Calibre's Quality Check sync to reconcile.
188
193
  - A per-book progress line goes to stderr (stdout stays a clean report); `--quiet` suppresses it. A corrupt or unreadable book is reported and skipped, never aborting the sweep.
@@ -0,0 +1,119 @@
1
+ # REPORT-12-Sept: the repair-landscape research (bindery-cli vs real-world EPUB damage)
2
+
3
+ Research date: 2026-09-10/11. Method: a read-only research pass mapping
4
+ the full taxonomy of real-world EPUB damage — by producer (Calibre
5
+ conversions, Word/Office export, InDesign, legacy OCR, Adobe Digital
6
+ Editions, KindleUnpack, self-pub aggregators, Windows case-manglers) —
7
+ against this repo's shipped repair surface, the Phase 14/15 backlogs,
8
+ and the Bug Reports. References include the upstream clone's format
9
+ readers. Full evidence lives in the agent's findings; this report is
10
+ the distilled verdict and every actionable gap with its routing.
11
+
12
+ ## Verdict
13
+
14
+ Bindery handles roughly **60-65% of real-world damage classes
15
+ end-to-end** and is near-complete for its founding niche: markup
16
+ well-formedness (the conversion-fatals class), NCX basics,
17
+ zip/mimetype container hygiene, dead-reference pruning, watermark and
18
+ page-number furniture, and archive/spine diagnosis (corruption, DRM
19
+ skip, ToC-bloat classification). Phase 14's hardening and all five Bug
20
+ Reports are shipped through v0.35.0; Phase 15 (the completeness
21
+ analyzer) is approved.
22
+
23
+ The missing ~35% is not more markup surgery. It is **package-level
24
+ repairs** — container.xml, manifest media-types, spine attribute
25
+ cleanup, encryption.xml/obfuscation state, zip-level dedupe, href
26
+ case/IRI mismatches — and the finding that matters most: several of
27
+ those classes are **gateway defects**. Epubcheck stays fatal while
28
+ they persist, so no shipped repair can ever be gate-accepted on those
29
+ books. They are locked out of the pipeline entirely.
30
+
31
+ ## A. The damage-taxonomy matrix and the repair gaps
32
+
33
+ 1. **Missing/corrupt `META-INF/container.xml`** (any producer):
34
+ epubcheck fatal; the OPF is unfindable. The locator falls back to
35
+ the first `.opf` but never writes the file. Deterministic fix:
36
+ generate the standard container.xml at the located OPF (upstream
37
+ calibre's `initialize_container` is the model). Effort S; HIGH
38
+ value — a gateway repair that unlocks an entire class of books.
39
+ 2. **Stale font-obfuscation state** (publisher EPUB3 fonts; IDPF
40
+ `2008/embedding` and Adobe `pdf/enc#RC`): post-KindleUnpack/strip
41
+ tools decrypt fonts in place but leave `encryption.xml` declaring
42
+ obfuscation; epubcheck errors and the audit implies DRM where there
43
+ is none. Fix: detect via font magic bytes at offset 0 vs the
44
+ CipherReference; prune stale entries; drop encryption.xml when only
45
+ obfuscation entries remain and the fonts verify. Never
46
+ un-obfuscate. Also gives the audit a real OBFUSCATED-vs-DRM
47
+ distinction. Effort M; needs the magic-byte detection validated on
48
+ real samples first.
49
+ 3. **Wrong/missing manifest `media-type`** (aggregators, ADE): fonts,
50
+ images, and media declared with wrong types. Fix: sniff magic bytes
51
+ or extension map; attribute-only. Effort S.
52
+ 4. **href case/backslash/IRI mismatches** (KindleUnpack, Windows
53
+ producers): `Images/Cover.JPG` vs `images/cover.jpg`, backslash
54
+ separators, `%XX` variants — the file IS present, so
55
+ `--prune-missing-resources` cannot touch the RSC-007/PKG-010 it
56
+ causes. Fix: case-insensitive namemap when exactly one match;
57
+ backslash to slash. Effort M.
58
+ 5. **Dangling reference edges after pruning** (any producer):
59
+ `spine@toc`, `item@media-overlay`, `item@fallback`, and EPUB2 cover
60
+ meta pointing at items the prune pass deleted. Today's prune checks
61
+ only `spine_ids` — it can manufacture the regression the gate then
62
+ rejects. Fix: extend prune to rewrite those edges (the rewrite
63
+ machinery exists in fix_manifest_ids). Effort S/M.
64
+ 6. **Duplicate zip entries** (any producer): last-wins shadowing; the
65
+ audit counts `dup_entries`, the rewrite copies both. Fix: drop
66
+ shadowed duplicates on rewrite. Effort S.
67
+ 7. **Cover wiring** (EPUB2 producers): dangling `<meta name="cover">`,
68
+ missing EPUB3 `properties="cover-image"`, wrong cover media-type —
69
+ calibre and booksellers extract no cover. Narrow, finding-driven
70
+ repair when the cover is identifiable from the existing guide/meta;
71
+ audit-only otherwise. Effort M.
72
+ 8. **Broken CSS from converters**: `url()` to absent assets, malformed
73
+ selectors. Repair is fiddly; ship the audit-only detector first and
74
+ let prevalence data decide. Effort M (repair).
75
+ 9. **Human-facing metadata validity** (self-pub/Word): non-ISO
76
+ `dc:date`, invalid `dc:language`, missing EPUB3
77
+ `dcterms:modified`. Explicit non-goal today; `dcterms:modified`
78
+ also collides with byte-determinism. Needs a written charter
79
+ ruling: carve-out with a determinism-safe timestamp policy, or
80
+ route to cquarry/CalibreQuarry. Effort M + a decision.
81
+ 10. **EPUB3 dual-ToC drift** (nav vs NCX): detection is a
82
+ deterministic structural diff feeding `decisions_needed`; full
83
+ sync is ToC synthesis and stays out of charter permanently.
84
+ Effort L if repair is ever attempted; audit-only recommended.
85
+ 11. **No-repair classes, correctly routed**: OCR garbage, truncated
86
+ re-sources, absent font families (audit + re-source); real DRM
87
+ (clean ENCRYPTED skip — shipped); DB-level format rows
88
+ (cquarry's, done); catalog completeness and human-facing metadata
89
+ quality (CalibreQuarry's audits).
90
+
91
+ ## B. Research-before-boxing
92
+
93
+ 1. **FastSweep prevalence study** — run `fast_sweep.py --mode=extract`
94
+ aggregation over the full library targeting the new classes
95
+ (container.xml presence/validity, encryption.xml algorithms,
96
+ media-type warnings, case-mismatch RSC-007s). The harness exists;
97
+ it sizes every box above with real counts before any is built.
98
+ 2. **The metadata carve-out ruling** (gap 9) — Brandon decides;
99
+ the audit records both options.
100
+ 3. **The obfuscation magic-byte table** (gap 2) — validate IDPF and
101
+ Adobe font prefixes against real samples before the repair ships.
102
+
103
+ ## C. Routing notes
104
+
105
+ Package-structure repairs (items 1-6) fit the existing "structural
106
+ opt-ins + gate" model without reopening the semantics-preserving
107
+ contract. Items 7-10 each need an explicit scope decision first.
108
+ Cross-repo: DB-level size/format-row mismatches route to cquarry
109
+ (set_format, shipped); catalog completeness, duplicate detection, and
110
+ human-facing metadata quality route to CalibreQuarry's audits; OCR,
111
+ truncation, monolithic, and missing-font content stay audit-and-
112
+ re-source permanently.
113
+
114
+ ## D. Phase 16 routing
115
+
116
+ Items 1-6 (the ungated package-structure repairs) are boxed in
117
+ roadmap.md **Phase 16** with the FastSweep prevalence study leading.
118
+ Items 7-10 are recorded as decision-gated boxes. Phase 15 (approved)
119
+ runs first if its fixture work is not yet done.
@@ -1,4 +1,56 @@
1
1
  # bindery-cli Patch Notes
2
+ ## v0.36.0 (2026-09-12)
3
+
4
+ ### The completeness spot-check: `audit completeness` (Phase 15)
5
+
6
+ - **The phase-1 judgment step finally has a tool owner.** The import
7
+ skill's completeness step asks a human question with no tool behind
8
+ it: "sample early, middle, and late pages ... read the LAST content
9
+ page to confirm it reaches real back matter rather than cutting off
10
+ mid-chapter". Two consecutive phase-1 runs answered it with a
11
+ hand-rolled zipfile sampler in /tmp. The new analyzer rides the
12
+ audit's existing single decompression pass (no second decode of any
13
+ book) and reports, per book: spine doc count, prose-doc count (spine
14
+ docs with at least 400 visible characters), the opening and closing
15
+ 120 characters of the first/middle/last prose doc, a trailing-ToC
16
+ classification of the final spine doc, and the fraction of unreadable
17
+ docs (corrupt entries plus unresolved itemrefs).
18
+ - **A trailing ToC never impersonates the ending.** The final spine doc
19
+ is classified as a chapter list when short link lines dominate it and
20
+ no block runs to paragraph length; such a doc is book furniture and
21
+ is excluded from the prose sampling, so the closing excerpt comes
22
+ from the real back matter. Verified against the real 2026-09-10
23
+ Redwall fixtures: Lord Brocktree's trailing ToC is caught and the
24
+ Epilogue is sampled as the last prose doc; The Long Patrol reads
25
+ exactly 55/57 prose docs through the percent-encoded hrefs that
26
+ defeated the hand-rolled sampler; Mattimeo's split doc (chapter
27
+ headings with real prose after every one) stays prose.
28
+ - **Advisory by contract.** The analyzer never flags a book and never
29
+ moves the exit code: the archive, spine, and emptytext verdicts keep
30
+ owning the flags. The verdict status is ADVISORY (not OK) when a
31
+ trailing ToC is present or the unreadable fraction reaches 10%.
32
+ `audit all` and `run phase1` include it through the battery, the CLI
33
+ grows the `completeness` mode, and `--json` records carry its
34
+ verdicts in the same shape as the other analyzers.
35
+
36
+ ### Dependencies
37
+
38
+ - Adopted cquarry 1.18.0 (floor + lock): the set_format repair lane now keeps
39
+ Calibre's FTS index and page counts honest automatically (the sidecar's
40
+ dirty queue and the pages rescan flag ride the write), and the search and
41
+ runtime fixes come along in the shared layer.
42
+ - Adopted cquarry 1.19.0 (floor + lock): the repair lane gains its undo
43
+ primitive -- save_original_format/restore_original_format -- so a format
44
+ repair can put the original bytes back if it goes wrong.
45
+
46
+ ### Repo records
47
+
48
+ - The release-tag backlog is settled forward-only from v0.35.0: the
49
+ nine untagged 0.18.0-0.26.0 entries are exempted by decision of
50
+ record (2026-09-11), and the v0.28.0 retag and the v0.33.0 em-dash
51
+ tag repair stay recorded-only force-pushes (roadmap.md, "Tag
52
+ policy").
53
+
2
54
  ## v0.35.0 (2026-09-10)
3
55
 
4
56
  ### The epubcheck daemon, reconciled: oracle-identical counts at ~15x the speed
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "bindery-cli"
7
- version = "0.35.0"
7
+ version = "0.36.0"
8
8
  description = "Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library."
9
9
  readme = "README.md"
10
10
  classifiers = [
@@ -27,7 +27,7 @@ authors = [
27
27
  dependencies = [
28
28
  "tqdm",
29
29
  "vir-tui>=2.2.0",
30
- "cquarry>=1.17.0",
30
+ "cquarry>=1.19.0",
31
31
  ]
32
32
 
33
33
  [project.optional-dependencies]
@@ -663,7 +663,7 @@ book instead raises an uncaught `TypeError` at the `book["path"]` access.
663
663
  matches the DB's book path and format name before the batch; otherwise save
664
664
  in place and warn without touching the row, and catch the `TypeError` path.
665
665
  *(Fixed in v0.32.0: a guessed id now drives a row update only when
666
- metadata.db corroborates it — the books row exists, the file sits in that
666
+ metadata.db corroborates it: the books row exists, the file sits in that
667
667
  book's catalogued directory, and when an EPUB row exists it carries that
668
668
  row's stored name. Any other guess (a stray file in a book directory, a
669
669
  stale `(N)` directory whose book is gone, the resolver's id gone from
@@ -996,7 +996,7 @@ can be net-neutral and ships silently.*
996
996
  v0.22.0 "its own verdict in every mode" promise (`audit.py:1565`,
997
997
  `1783-1803`, `1520-1558`).
998
998
  *(Done in v0.33.0: the archive and spine sections run unconditionally
999
- after the selected content sections — they are always-on verdicts and
999
+ after the selected content sections; they are always-on verdicts and
1000
1000
  were already scanned and recorded in the JSON; only the console/rc
1001
1001
  path was dead. A CRC-corrupt book now prints its CORRUPT section and
1002
1002
  fails library mode with exit 1, matching directory mode. Test pins
@@ -1286,7 +1286,7 @@ can be net-neutral and ships silently.*
1286
1286
 
1287
1287
  ### Documentation (drift is real but narrow)
1288
1288
 
1289
- - [ ] **Reconcile spec.md with the shipped opt-ins.** The contract's
1289
+ - [x] **Reconcile spec.md with the shipped opt-ins.** The contract's
1290
1290
  non-goals forbid exactly what `--reserialize` (v0.3.0, the lone
1291
1291
  html5lib dependency) and `--strip-bad-attrs` (v0.4.0) do, and neither
1292
1292
  has a spec section; the OPF half of `--fix-ids` is missing;
@@ -1294,6 +1294,10 @@ can be net-neutral and ships silently.*
1294
1294
  NCX sentence at spec.md:52-53 is garbled (a dangling "to the OPF
1295
1295
  unique identifier" tail). A contributor reading spec.md first would
1296
1296
  conclude two shipped flags violate the charter.
1297
+ *(APPLIED 2026-09-10, commit 01ad173, on Brandon's verbatim go
1298
+ recorded in the audit sheet's evening answers; shipped with
1299
+ v0.34.0. Ticked 2026-09-11 in the main-thread double-check after
1300
+ the verifier caught the work shipping unticked.)*
1297
1301
  **GATED on Brandon: spec.md is the contract file. Draft proposal,
1298
1302
  ready to apply on a go (do not apply without one):**
1299
1303
  (1) Non-goals bullet 2 ("Repairing genuinely mangled structure
@@ -1368,6 +1372,14 @@ everything else is backlog, not danger.*
1368
1372
 
1369
1373
  ## Phase 15: completeness spot-check analyzer (proposed 2026-09-10, from the phase-1 skill's own doctrine)
1370
1374
 
1375
+ *(SHIPPED 2026-09-12 as v0.36.0: `audit completeness` rides the existing
1376
+ single decompression pass; the Redwall-shaped regression tests landed
1377
+ first (4021ce7), the analyzer second (e6156c0). Verified against the
1378
+ real fixtures: Lord Brocktree's trailing ToC is caught with the Epilogue
1379
+ sampled as the last prose doc, The Long Patrol reads exactly 55/57
1380
+ prose docs through its percent-encoded hrefs, and Mattimeo's split doc
1381
+ stays prose. Advisory by contract; `all` and `run phase1` include it.)*
1382
+
1371
1383
  *(RAISED with Brandon 2026-09-10, batched with the session-start raises;
1372
1384
  recommendation: APPROVE. Execution is gated on that go; the proposal
1373
1385
  below is filed and costed. The Redwall run it names is a ready-made
@@ -1392,3 +1404,88 @@ Brocktree (real Epilogue doc before a trailing 520-char ToC), Mattimeo
1392
1404
  (Chapter 50 verified as a real heading with prose after, inside a 174k-char
1393
1405
  split doc), The Long Patrol (percent-encoded hrefs defeated the hand-rolled
1394
1406
  reader; 55/57 prose docs).*
1407
+
1408
+ ## Phase 16: package-structure repairs (proposed 2026-09-12, from REPORT-12-Sept.md)
1409
+
1410
+ A research pass mapped the full real-world EPUB damage taxonomy against
1411
+ the shipped repair surface: bindery handles ~60-65% of damage classes
1412
+ end-to-end, and the missing slice is package-level structure — several
1413
+ classes of which are GATEWAY DEFECTS (epubcheck stays fatal, so no
1414
+ shipped repair can ever be gate-accepted on those books). Full evidence,
1415
+ producer classes, and the cross-repo routing live in
1416
+ REPORT-12-Sept.md. Lead with the prevalence study; every box below is
1417
+ an opt-in structural repair under the existing gate contract.
1418
+
1419
+ ### A. Research first (boxes cannot be honestly sized without these)
1420
+
1421
+ - [ ] **FastSweep prevalence study**: run `fast_sweep.py
1422
+ --mode=extract` aggregation over the full library for the new
1423
+ classes — container.xml presence/validity, encryption.xml
1424
+ algorithms, manifest media-type warnings, case-mismatch RSC-007s.
1425
+ Each Phase 16 repair box gets its real count before it opens.
1426
+ - [ ] **Obfuscation magic-byte validation**: confirm the IDPF
1427
+ (`2008/embedding`) and Adobe (`pdf/enc#RC`) font prefixes at offset 0
1428
+ against real samples before the encryption-repair box opens.
1429
+
1430
+ ### B. The repairs (each: opt-in flag, dry-run default, gate-accepted)
1431
+
1432
+ - [ ] **`--fix-container`**: generate META-INF/container.xml at the
1433
+ located OPF (model: upstream calibre's initialize_container).
1434
+ Gateway repair, effort S, HIGH value.
1435
+ - [ ] **Obfuscation-aware encryption.xml repair**: prune stale
1436
+ obfuscation entries when font magic bytes prove the fonts are
1437
+ decrypted in place; drop encryption.xml when only obfuscation entries
1438
+ remain and fonts verify. NEVER un-obfuscate. M.
1439
+ - [ ] **Audit distinction: OBFUSCATED vs DRM** in audit.py (today the
1440
+ ENCRYPTED verdict keys on unreadable entries and implies DRM where
1441
+ publisher obfuscation is benign). S, rides the box above.
1442
+ - [ ] **Reference-edge completion for `--prune-missing-resources`**:
1443
+ rewrite `spine@toc`, `item@media-overlay`, `item@fallback`, and EPUB2
1444
+ cover-meta edges when their targets are pruned (machinery exists in
1445
+ fix_manifest_ids). Prevents manufacturing gate regressions. S/M.
1446
+ - [ ] **Manifest media-type normalization**: sniff magic bytes /
1447
+ extension map; attribute-only. S.
1448
+ - [ ] **href case/backslash resolution**: case-insensitive namemap when
1449
+ exactly one zip entry matches; backslash to slash. Kills the
1450
+ present-file RSC-007/PKG-010 slice prune cannot touch. M.
1451
+ - [ ] **Duplicate zip-entry dedupe on rewrite**: drop shadowed
1452
+ duplicates (audit already counts `dup_entries`). S.
1453
+
1454
+ ### C. Decision-gated (recorded boxes; no work without a ruling)
1455
+
1456
+ - [ ] **Cover wiring repair** (dangling `<meta name="cover">`, missing
1457
+ `properties="cover-image"`, wrong cover media-type): narrow,
1458
+ finding-driven repair vs audit-only. Needs the scope decision. M.
1459
+ - [ ] **Metadata validity carve-out ruling** (non-ISO `dc:date`,
1460
+ invalid `dc:language`, missing `dcterms:modified`): bindery opt-in
1461
+ with a determinism-safe timestamp policy, or route to
1462
+ cquarry/CalibreQuarry. Needs a written charter ruling. M.
1463
+ - [ ] **NCX<->nav drift DETECTOR** (audit-only; structural diff feeding
1464
+ `decisions_needed`; ToC synthesis stays out of charter
1465
+ permanently). L if repair is ever attempted.
1466
+ - [ ] **CSS url() pruning** as `--prune-missing-resources` scope
1467
+ extension: only after the prevalence study says it matters. M.
1468
+
1469
+ Ship shape: A leads; then B in value order (container, encryption,
1470
+ prune-edges, media-types); C boxes record their decisions and wait.
1471
+ Each repair ships with a fixture built from the prevalence study's
1472
+ real samples, a regression test written before the fix, and the gate
1473
+ (epubcheck) accepting the result.
1474
+
1475
+ ## Tag policy (decided 2026-09-11, recorded 2026-09-12)
1476
+
1477
+ The release-tag backlog is settled, forward-only:
1478
+
1479
+ - **v0.35.0 is the anchor.** Every release from v0.36.0 on is tagged at
1480
+ its release commit under auto-tag promotion (a fully qualified release
1481
+ is tagged and pushed without asking; the blitz rulebook's decision 22).
1482
+ - **The nine untagged 0.18.0-0.26.0 entries are exempted** by decision
1483
+ of record (2026-09-11): no backfill tags will be cut for them. Their
1484
+ release notes live in patchnotes.md and their wheels are on PyPI; the
1485
+ missing git tags are accepted history.
1486
+ - **Two recorded-only repairs stay force-pushes, undecided:** the
1487
+ v0.28.0 retag (the tag points at a red commit) and the v0.33.0
1488
+ em-dash tag message (the pushed tag carries two em-dashes the
1489
+ proofread missed; the source patchnotes entry was fixed immediately in
1490
+ 1c374c1). Either would need publish.yml disabled around the push and
1491
+ Brandon's explicit go; neither is ever auto-executed.
@@ -243,8 +243,8 @@ daemon (`_EpubcheckDaemon`, a bounded pool sized by `--workers`) that writes
243
243
  `FastDaemon.java` at runtime and launches it through Java's single-file source
244
244
  launcher (in-memory compilation by the running JVM: no javac, no version skew),
245
245
  driving the warm JVM over a pipe. The daemon answers with the counts it reads from
246
- the JSON that epubcheck's own `CheckingReport.generate()` serializes — the same
247
- document `--json` produces — so both paths measure identically by construction.
246
+ the JSON that epubcheck's own `CheckingReport.generate()` serializes (the same
247
+ document `--json` produces), so both paths measure identically by construction.
248
248
  (Epubcheck's JSON counts aggregated messages, not occurrences: a defect repeated
249
249
  three times is one error. The gate has always measured that scale.) Every daemon
250
250
  roundtrip is bounded by the caller's timeout, and any daemon failure tears it down
@@ -343,8 +343,8 @@ crashing the sweep.
343
343
 
344
344
  ## Audit subcommand (read-only)
345
345
 
346
- `bindery audit {content,pagenumbers,emptytext,ocr,monolithic,all} [PATH] [--max-doc-chars N]
347
- [--tag TAG] [--id IDs]` (v0.15.0, `audit.py`; `--tag` since v0.18.0; `monolithic` since v0.21.0; `--id` since v0.19.0, comma-lists in v0.23.0) inspects
346
+ `bindery audit {content,pagenumbers,emptytext,ocr,monolithic,completeness,all} [PATH] [--max-doc-chars N]
347
+ [--tag TAG] [--id IDs]` (v0.15.0, `audit.py`; `--tag` since v0.18.0; `monolithic` since v0.21.0, `--max-doc-chars N`; `--id` since v0.19.0, comma-lists in v0.23.0; `completeness` since v0.36.0) inspects
348
348
  EPUB body text for flaws epubcheck cannot see: non-English script blocks, baked-in page-number
349
349
  layers (sliding-window density heuristics), empty or thin books, systemic OCR damage, and
350
350
  single oversized content documents (one spine doc at or above 300k characters — readers refuse
@@ -353,6 +353,18 @@ is fully read for CRC + decompression, reporting CORRUPT rather than EMPTY), and
353
353
  issues. Manifest/NCX references to absent files are classified as either `convention` (ToC is bloated but
354
354
  present documents form a consecutive chapter span) or `fragment` (the span itself is broken).
355
355
 
356
+ The completeness analyzer is the phase-1 spot-check and is advisory by contract: it never
357
+ flags a book and never moves the exit code. Per book it reports the spine doc count, the
358
+ prose-doc count (spine docs carrying at least 400 visible characters), the opening and closing
359
+ 120 characters of the first/middle/last prose doc, a trailing-ToC classification of the final
360
+ spine doc, and the fraction of spine docs that could not be read (corrupt entries plus
361
+ unresolved itemrefs). A trailing ToC (the final doc is dominated by short link lines, at
362
+ least half its lines carry links, and no block runs to paragraph length) is book furniture:
363
+ it is excluded from the prose sampling so the closing excerpt comes from the real back
364
+ matter, and the verdict becomes ADVISORY (as it does at an unreadable fraction of 10% or
365
+ more). Like emptytext it is skipped wholesale when the archive verdict owns the body-text
366
+ story.
367
+
356
368
  In library mode, EPUB files are resolved through `cquarry.db.CalibreDB.get_format_path()` — the
357
369
  storage-layout logic is not duplicated here. The scan itself still writes nothing. The opt-in
358
370
  `--tag TAG` pass applies `TAG` to flagged books via `cquarry.write.WritableCalibreDB`, the
@@ -4,7 +4,7 @@ strip), epubcheck-gated.
4
4
  Single source of truth for the version. pyproject.toml must match VERSION below.
5
5
  """
6
6
 
7
- VERSION = "0.35.0"
7
+ VERSION = "0.36.0"
8
8
  __version__ = VERSION
9
9
 
10
10
  from .epub import RepairReport, repair_epub # noqa: E402
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env python3
2
2
  """
3
3
  bindery audit: read the actual text of every EPUB and flag content problems that
4
- metadata and structural validators cannot see. Four analyzers, one tool:
4
+ metadata and structural validators cannot see. Six analyzers, one tool:
5
5
 
6
6
  content non-English bodies (wrong-language editions) and injected
7
7
  foreign-language ad-notices (declared lang=eng, body Portuguese
@@ -20,7 +20,14 @@ metadata and structural validators cannot see. Four analyzers, one tool:
20
20
  their hyphenated form). Character-substitution errors ("sonic"
21
21
  for "some") are OUT OF SCOPE: catching those needs a wordlist,
22
22
  and this tool is stdlib-only by contract.
23
- all run all four in a SINGLE decompression pass per book
23
+ monolithic one spine doc so large that readers refuse to render it, though
24
+ the book totals normally
25
+ completeness the phase-1 spot-check: prose-doc count against the spine, the
26
+ opening/closing text of the first/middle/last prose doc, a
27
+ trailing-ToC classification of the final doc, and the fraction
28
+ of unreadable docs (advisory: it reports the shape a human
29
+ judges, it never flags the book)
30
+ all run all six in a SINGLE decompression pass per book
24
31
 
25
32
  This merges the former audit_epub_content.py / audit_epub_pagenumbers.py /
26
33
  audit_epub_emptytext.py: they shared the same spine resolution, library/
@@ -1127,6 +1134,132 @@ def scan_monolithic(path: Path) -> dict:
1127
1134
  return analyze_monolithic(load_book(path))
1128
1135
 
1129
1136
 
1137
+ # ----------------------------------------------------------------------------
1138
+ # Analyzer: completeness (the phase-1 spot-check: does the book reach its end?)
1139
+ # ----------------------------------------------------------------------------
1140
+
1141
+ # A spine doc at or above this many visible chars is real content; under it is
1142
+ # front/back matter (title pages, colophons) or furniture.
1143
+ PROSE_DOC_MIN_CHARS = 400
1144
+ SPOT_CHARS = 120 # opening/closing excerpt length per sampled doc
1145
+ # A book this far unreadable (corrupt entries + unresolved itemrefs against
1146
+ # all spine itemrefs) draws the advisory, though it never fails a run.
1147
+ COMPLETENESS_UNREADABLE_FLAG = 0.10
1148
+
1149
+ # Trailing-ToC detection, tuned on the 2026-09-10 Redwall run: Lord
1150
+ # Brocktree's trailing ToC is a 520-char link list sitting AFTER a real
1151
+ # Epilogue doc, while Mattimeo's final doc carries chapter headings with real
1152
+ # prose after every one of them and must never classify as a ToC. A ToC page
1153
+ # is dense with short link lines and carries no real paragraphs.
1154
+ TRAILING_TOC_MIN_BLOCKS = 4 # fewer lines is a colophon or pager, not a ToC
1155
+ TRAILING_TOC_SHORT_MAX = 100 # a chapter-heading line is short
1156
+ TRAILING_TOC_SHORT_FRAC = 0.75 # most lines are short
1157
+ TRAILING_TOC_LONG_BLOCK = 400 # a real paragraph runs longer than any ToC line
1158
+ TRAILING_TOC_LINK_FRAC = 0.5 # at least half the lines carry a link
1159
+
1160
+ _TOC_BLOCK_SPLIT_RE = re.compile(r"</(?:p|div|li|h[1-6]|td)>", re.IGNORECASE)
1161
+ _A_TAG_RE = re.compile(r"<a[\s>]", re.IGNORECASE)
1162
+
1163
+
1164
+ def _trailing_toc(html_src: str) -> bool:
1165
+ """Is this document a chapter list? Short link lines dominate and no
1166
+ block runs to paragraph length; a heading followed by real prose (the
1167
+ Mattimeo split-doc case) keeps a document out."""
1168
+ body = _SCRIPT_STYLE_RE.sub(" ", html_src)
1169
+ blocks: list[str] = []
1170
+ for chunk in _TOC_BLOCK_SPLIT_RE.split(body):
1171
+ text = _visible_text(chunk)
1172
+ if text:
1173
+ blocks.append(text)
1174
+ if len(blocks) < TRAILING_TOC_MIN_BLOCKS:
1175
+ return False
1176
+ short = sum(1 for b in blocks if len(b) <= TRAILING_TOC_SHORT_MAX)
1177
+ if short / len(blocks) < TRAILING_TOC_SHORT_FRAC:
1178
+ return False
1179
+ if any(len(b) > TRAILING_TOC_LONG_BLOCK for b in blocks):
1180
+ return False
1181
+ links = len(_A_TAG_RE.findall(body))
1182
+ return links >= len(blocks) * TRAILING_TOC_LINK_FRAC
1183
+
1184
+
1185
+ def analyze_completeness(book: Book) -> dict:
1186
+ """The phase-1 completeness spot-check, over the (pre-read) spine.
1187
+
1188
+ Reports the shape a human needs in order to judge "does this book reach
1189
+ its end": the prose-doc count against the spine, opening/closing excerpts
1190
+ of the first/middle/last prose doc, a trailing-ToC classification of the
1191
+ final doc, and the fraction of spine docs that could not be read at all
1192
+ (corrupt entries plus unresolved itemrefs). A trailing ToC is book
1193
+ furniture: it is excluded from the prose sampling so the closing excerpt
1194
+ comes from the real back matter (Lord Brocktree's Epilogue, not its
1195
+ 520-char ToC). Reuses the shared visible_texts() cache: no second pass.
1196
+ """
1197
+ texts = book.visible_texts()
1198
+ spine_n = len(book.spine)
1199
+ trailing = ""
1200
+ if spine_n >= 2 and _trailing_toc(book.docs.get(book.spine[-1], "")):
1201
+ trailing = book.spine[-1]
1202
+ prose = [
1203
+ (book.spine[i], texts[i])
1204
+ for i in range(spine_n)
1205
+ if len(texts[i]) >= PROSE_DOC_MIN_CHARS and book.spine[i] != trailing
1206
+ ]
1207
+ corrupt_set = set(book.corrupt)
1208
+ unreadable = (
1209
+ sum(1 for i in range(spine_n) if not texts[i] and book.spine[i] in corrupt_set)
1210
+ + book.spine_missing
1211
+ )
1212
+ blank_docs = sum(
1213
+ 1 for i in range(spine_n) if not texts[i] and book.spine[i] not in corrupt_set
1214
+ )
1215
+ spots: list[dict] = []
1216
+ if prose:
1217
+ picks = sorted({0, len(prose) // 2, len(prose) - 1})
1218
+ spots = [
1219
+ {
1220
+ "doc": prose[i][0],
1221
+ "chars": len(prose[i][1]),
1222
+ "opens": prose[i][1][:SPOT_CHARS],
1223
+ "ends": prose[i][1][-SPOT_CHARS:],
1224
+ }
1225
+ for i in picks
1226
+ ]
1227
+ total = spine_n + book.spine_missing
1228
+ return {
1229
+ "spine_docs": spine_n,
1230
+ "spine_missing": book.spine_missing,
1231
+ "prose_docs": len(prose),
1232
+ "unreadable": unreadable,
1233
+ "unreadable_frac": (unreadable / total) if total else 0.0,
1234
+ "blank_docs": blank_docs,
1235
+ "trailing_toc": trailing,
1236
+ "spots": spots,
1237
+ }
1238
+
1239
+
1240
+ def scan_completeness(path: Path) -> dict:
1241
+ return analyze_completeness(load_book(path))
1242
+
1243
+
1244
+ def _completeness_dir(r: dict) -> tuple[bool, str, list[str]]:
1245
+ """Advisory by contract: the archive, spine, and emptytext verdicts own
1246
+ the flags; completeness reports the shape a human judges."""
1247
+ lines = [
1248
+ f"prose {r['prose_docs']}/{r['spine_docs']} spine docs, "
1249
+ f"unreadable {r['unreadable']} ({r['unreadable_frac'] * 100:.0f}%)"
1250
+ ]
1251
+ status = "OK"
1252
+ if r["trailing_toc"]:
1253
+ lines.append(f"trailing ToC: {r['trailing_toc']}")
1254
+ status = "ADVISORY"
1255
+ elif r["unreadable_frac"] >= COMPLETENESS_UNREADABLE_FLAG:
1256
+ status = "ADVISORY"
1257
+ if r["blank_docs"]:
1258
+ lines.append(f"{r['blank_docs']} spine doc(s) present but blank")
1259
+ lines += [f"{s['doc']}: [{s['opens']}] ... [{s['ends']}]" for s in r["spots"]]
1260
+ return (False, status, lines)
1261
+
1262
+
1130
1263
  # ----------------------------------------------------------------------------
1131
1264
  # Analyzer: archive corruption (always on; owns the "empty body" story)
1132
1265
  # ----------------------------------------------------------------------------
@@ -1682,11 +1815,40 @@ def _spine_sections(hits, advisory) -> int:
1682
1815
  return 0
1683
1816
 
1684
1817
 
1818
+ def _completeness_sections(advisory) -> int:
1819
+ """Print the completeness advisories; the analyzer never fails a run."""
1820
+ if advisory:
1821
+ print(
1822
+ f"{YELLOW}{BOLD}COMPLETENESS SPOT-CHECK ({len(advisory)} to eyeball;"
1823
+ f" reported, not flagged){RESET}"
1824
+ )
1825
+ for book_id, title, tag, r in sorted(advisory):
1826
+ print(f" {YELLOW}#{book_id}{RESET} [{tag}] {title}")
1827
+ _p, _s, lines = _completeness_dir(r)
1828
+ for ln in lines:
1829
+ print(f" {ln}")
1830
+ print()
1831
+ print(
1832
+ f"{YELLOW}{BOLD}completeness DONE{RESET}: read the excerpts above;"
1833
+ f" a trailing ToC or unreadable docs never fail the run."
1834
+ )
1835
+ return 0
1836
+ print(f"{GREEN}{BOLD}completeness CLEAN{RESET}: every book reads whole.")
1837
+ return 0
1838
+
1839
+
1685
1840
  # ----------------------------------------------------------------------------
1686
1841
  # Runner
1687
1842
  # ----------------------------------------------------------------------------
1688
1843
 
1689
- ALL: tuple[str, ...] = ("content", "pagenumbers", "emptytext", "ocr", "monolithic")
1844
+ ALL: tuple[str, ...] = (
1845
+ "content",
1846
+ "pagenumbers",
1847
+ "emptytext",
1848
+ "ocr",
1849
+ "monolithic",
1850
+ "completeness",
1851
+ )
1690
1852
 
1691
1853
 
1692
1854
  def _record_verdict(
@@ -1813,6 +1975,7 @@ def run_library(
1813
1975
  thin_hits: list[tuple] = []
1814
1976
  ocr_found: list[tuple] = []
1815
1977
  mono_hits: list[tuple] = []
1978
+ completeness_advisory: list[tuple] = []
1816
1979
  corrupt_hits: list[tuple] = []
1817
1980
  spine_hits: list[tuple] = []
1818
1981
  spine_advisory: list[tuple] = []
@@ -1908,6 +2071,15 @@ def run_library(
1908
2071
  mono_hits.append((book_id, title, tag, r))
1909
2072
  _record_verdict(record, "monolithic", _monolithic_dir(r, max_doc_chars))
1910
2073
 
2074
+ if "completeness" in selected and not corrupt_r["n"]:
2075
+ r = analyze_completeness(book)
2076
+ _p, status, lines = _completeness_dir(r)
2077
+ if status != "OK":
2078
+ completeness_advisory.append((book_id, title, tag, r))
2079
+ # advisory by contract: problem is always False, so the record
2080
+ # status and the exit code are never moved by this analyzer
2081
+ _record_verdict(record, "completeness", (False, status, lines))
2082
+
1911
2083
  print(f"Scanned {scanned} EPUBs in {library_root}\n")
1912
2084
  rc = 0
1913
2085
  multi = len(selected) > 1
@@ -1924,6 +2096,8 @@ def run_library(
1924
2096
  rc |= _empty_sections(empty_hits, partial_hits, thin_hits)
1925
2097
  elif key == "monolithic":
1926
2098
  rc |= _monolithic_sections(mono_hits)
2099
+ elif key == "completeness":
2100
+ rc |= _completeness_sections(completeness_advisory)
1927
2101
  else:
1928
2102
  rc |= _ocr_sections(ocr_found)
1929
2103
  if multi:
@@ -2104,7 +2278,7 @@ def run_directory(
2104
2278
  for key in ALL:
2105
2279
  if key not in selected:
2106
2280
  continue
2107
- if key == "emptytext" and corrupt_r["n"]:
2281
+ if key in ("emptytext", "completeness") and corrupt_r["n"]:
2108
2282
  continue # the archive verdict owns this book's body-text story
2109
2283
  if key == "content":
2110
2284
  problem, status, lines = _content_dir(analyze_content(book))
@@ -2118,6 +2292,8 @@ def run_directory(
2118
2292
  problem, status, lines = _monolithic_dir(
2119
2293
  analyze_monolithic(book), max_doc_chars
2120
2294
  )
2295
+ elif key == "completeness":
2296
+ problem, status, lines = _completeness_dir(analyze_completeness(book))
2121
2297
  else:
2122
2298
  problem, status, lines = _ocr_dir(analyze_ocr(book))
2123
2299
  if problem:
@@ -2272,7 +2448,7 @@ def run_single(
2272
2448
  for key in ALL:
2273
2449
  if key not in selected:
2274
2450
  continue
2275
- if key == "emptytext" and corrupt_r["n"]:
2451
+ if key in ("emptytext", "completeness") and corrupt_r["n"]:
2276
2452
  # Corruption owns the body-text story for this book; reporting
2277
2453
  # EMPTY here would be the wrong disease.
2278
2454
  continue
@@ -2288,6 +2464,8 @@ def run_single(
2288
2464
  problem, status, lines = _monolithic_dir(
2289
2465
  analyze_monolithic(book), max_doc_chars
2290
2466
  )
2467
+ elif key == "completeness":
2468
+ problem, status, lines = _completeness_dir(analyze_completeness(book))
2291
2469
  else:
2292
2470
  problem, status, lines = _ocr_dir(analyze_ocr(book))
2293
2471
  if problem:
@@ -808,8 +808,9 @@ def _phase1_status(audit_rec: dict | None, repair_rec: dict | None) -> str:
808
808
 
809
809
  def _run_phase1_audit(root: Path) -> tuple[list[dict], int]:
810
810
  """Stage 1: the read-only audit battery (corruption sweep, content
811
- battery, monolithic) exactly as `bindery audit all DIR` runs it, with the
812
- report captured through its own --json payload in a temp file."""
811
+ battery, monolithic, completeness spot-check) exactly as
812
+ `bindery audit all DIR` runs it, with the report captured through its
813
+ own --json payload in a temp file."""
813
814
  with tempfile.TemporaryDirectory() as td:
814
815
  out = Path(td) / "audit.json"
815
816
  rc = run_directory(
@@ -900,7 +901,7 @@ def run_phase1(args) -> int:
900
901
  print(f"Bindery run phase1 ({mode}): {root}\n")
901
902
 
902
903
  print(
903
- "== stage 1/2: audit battery (corruption sweep, content battery, monolithic) =="
904
+ "== stage 1/2: audit battery (corruption sweep, content battery, monolithic, completeness) =="
904
905
  )
905
906
  audit_records, audit_rc = _run_phase1_audit(root)
906
907
 
@@ -1357,7 +1358,15 @@ def build_parser() -> argparse.ArgumentParser:
1357
1358
  )
1358
1359
  audit.add_argument(
1359
1360
  "mode",
1360
- choices=("content", "pagenumbers", "emptytext", "ocr", "monolithic", "all"),
1361
+ choices=(
1362
+ "content",
1363
+ "pagenumbers",
1364
+ "emptytext",
1365
+ "ocr",
1366
+ "monolithic",
1367
+ "completeness",
1368
+ "all",
1369
+ ),
1361
1370
  help="which audit to run",
1362
1371
  )
1363
1372
  audit.add_argument(
@@ -1712,3 +1712,286 @@ class TestLibraryTagEndToEnd(unittest.TestCase):
1712
1712
  os.chdir(old)
1713
1713
  self.assertEqual(rc, 1) # an EMPTY book is a finding
1714
1714
  self.assertEqual(tagged, [(1,)])
1715
+
1716
+
1717
+ class TestCompletenessAnalyzer(unittest.TestCase):
1718
+ """Phase 15: the completeness spot-check analyzer. The fixture shapes come
1719
+ from the 2026-09-10 Redwall phase-1 run: Lord Brocktree (a real Epilogue
1720
+ doc sitting before a trailing 520-char ToC), Mattimeo (a 174k-char split
1721
+ doc whose final chapter heading has real prose after it), and The Long
1722
+ Patrol (percent-encoded hrefs the hand-rolled sampler could not read at
1723
+ all; 55/57 prose docs). Synthetic fillers replicate each SHAPE; no real
1724
+ text lives in this suite."""
1725
+
1726
+ CONTAINER = TestEmptyTextScan.CONTAINER
1727
+
1728
+ PARA = "The hares marched along the ridge in the fine morning light. "
1729
+
1730
+ def _prose(self, paras=7, heading=None, closer=""):
1731
+ h = f"<h2>{heading}</h2>" if heading else ""
1732
+ return h + "<p>" + (self.PARA * paras) + "</p>" + closer
1733
+
1734
+ def _epub(self, tmp, docs, hrefs=None):
1735
+ # docs: list of (name, body_html); hrefs optionally remaps the
1736
+ # manifest href (percent-encoded forms) while the archive name stays
1737
+ import zipfile as zf
1738
+
1739
+ manifest = ""
1740
+ for i, (n, _) in enumerate(docs):
1741
+ href = (hrefs or {}).get(n, n)
1742
+ manifest += (
1743
+ f'<item id="d{i}" href="{href}" media-type="application/xhtml+xml"/>'
1744
+ )
1745
+ spine = "".join(f'<itemref idref="d{i}"/>' for i in range(len(docs)))
1746
+ opf = (
1747
+ '<package xmlns="http://www.idpf.org/2007/opf">'
1748
+ f"<manifest>{manifest}</manifest><spine>{spine}</spine></package>"
1749
+ )
1750
+ p = pathlib.Path(tmp) / "t.epub"
1751
+ with zf.ZipFile(p, "w") as z:
1752
+ z.writestr("mimetype", "application/epub+zip")
1753
+ z.writestr("META-INF/container.xml", self.CONTAINER)
1754
+ z.writestr("content.opf", opf)
1755
+ for n, body in docs:
1756
+ z.writestr(n, f"<html><body>{body}</body></html>")
1757
+ return p
1758
+
1759
+ def test_real_back_matter_before_trailing_toc(self):
1760
+ # Lord Brocktree shape: six real chapters and a real Epilogue doc,
1761
+ # then a link-dense ToC (over the prose threshold at 456 visible
1762
+ # chars, like the real 520-char one) as the FINAL spine doc. The ToC
1763
+ # must classify as trailing furniture: never a prose sample, so the
1764
+ # closing excerpt comes from the Epilogue's back matter.
1765
+ chapters = [
1766
+ (f"ch{i}.xhtml", self._prose(7, f"Chapter {i}")) for i in range(1, 7)
1767
+ ]
1768
+ epilogue = (
1769
+ "epilogue.xhtml",
1770
+ self._prose(
1771
+ 7,
1772
+ "Epilogue",
1773
+ closer="<p>And so the tale closes with the hares home at last.</p>",
1774
+ ),
1775
+ )
1776
+ toc = (
1777
+ "toc.xhtml",
1778
+ "<h1>Contents</h1>"
1779
+ + "".join(
1780
+ f'<p><a href="ch{i}.xhtml">Chapter {i}: The March of the '
1781
+ "Hares and the Siege of the Mountain</a></p>"
1782
+ for i in range(1, 7)
1783
+ )
1784
+ + '<p><a href="epilogue.xhtml">Epilogue: Home at Salamandastron'
1785
+ "</a></p>",
1786
+ )
1787
+ docs = [("title.xhtml", "<p>A Tale of the Long Patrol</p>")]
1788
+ docs += chapters + [epilogue, toc]
1789
+ with tempfile.TemporaryDirectory() as tmp:
1790
+ r = audit.analyze_completeness(audit.load_book(self._epub(tmp, docs)))
1791
+ self.assertEqual(r["spine_docs"], 9)
1792
+ self.assertEqual(r["trailing_toc"], "toc.xhtml")
1793
+ # six chapters + the epilogue; the 456-char ToC is excluded
1794
+ self.assertEqual(r["prose_docs"], 7)
1795
+ self.assertEqual(r["spots"][-1]["doc"], "epilogue.xhtml")
1796
+ self.assertIn("home at last", r["spots"][-1]["ends"])
1797
+
1798
+ def test_final_split_doc_with_prose_after_heading_is_not_a_toc(self):
1799
+ # Mattimeo shape: one huge split doc carrying fifty chapter headings,
1800
+ # each followed by real prose (Chapter 50 was verified to have prose
1801
+ # after it). The link-line heuristics must never read it as a ToC,
1802
+ # and the closing excerpt must be the prose after the last heading.
1803
+ chapters = [
1804
+ (f"ch{i}.xhtml", self._prose(7, f"Chapter {i}")) for i in range(1, 6)
1805
+ ]
1806
+ split_body = "".join(
1807
+ f"<h2>Chapter {i}</h2><p>{self.PARA * 4}</p>" for i in range(1, 51)
1808
+ )
1809
+ split = ("split.xhtml", split_body)
1810
+ docs = [("title.xhtml", "<p>A Tale of the Long Patrol</p>")]
1811
+ docs += chapters + [split]
1812
+ with tempfile.TemporaryDirectory() as tmp:
1813
+ r = audit.analyze_completeness(audit.load_book(self._epub(tmp, docs)))
1814
+ self.assertEqual(r["trailing_toc"], "")
1815
+ self.assertEqual(r["spine_docs"], 7)
1816
+ self.assertEqual(r["prose_docs"], 6) # five chapters + the split doc
1817
+ self.assertEqual(r["spots"][-1]["doc"], "split.xhtml")
1818
+ self.assertIn("fine morning light", r["spots"][-1]["ends"])
1819
+
1820
+ def test_percent_encoded_hrefs_resolve(self):
1821
+ # The Long Patrol shape: manifest hrefs carry %20 runs for the spaces
1822
+ # in the stored archive names (the hand-rolled sampler read none of
1823
+ # these docs). The shipped audit's IRI decoding resolves them for
1824
+ # free: every spine doc lands, short front matter stays out of the
1825
+ # prose set, and the final doc is sampled whole.
1826
+ docs = [
1827
+ ("title.xhtml", "<p>A Tale of the Long Patrol</p>"),
1828
+ ("note.xhtml", "<p>A brief historical note.</p>"),
1829
+ ]
1830
+ docs += [
1831
+ (f"Chapter {i:02d}.xhtml", self._prose(7, f"Chapter {i}"))
1832
+ for i in range(1, 6)
1833
+ ]
1834
+ docs.append(("The End.xhtml", self._prose(7, "Chapter Six")))
1835
+ hrefs = {n: n.replace(" ", "%20") for n, _ in docs}
1836
+ with tempfile.TemporaryDirectory() as tmp:
1837
+ r = audit.analyze_completeness(
1838
+ audit.load_book(self._epub(tmp, docs, hrefs))
1839
+ )
1840
+ self.assertEqual(r["spine_docs"], 8)
1841
+ self.assertEqual(r["prose_docs"], 6)
1842
+ self.assertEqual(r["trailing_toc"], "")
1843
+ self.assertEqual(r["spots"][-1]["doc"], "The End.xhtml")
1844
+ self.assertEqual(len(r["spots"]), 3)
1845
+ self.assertTrue(r["spots"][0]["opens"])
1846
+
1847
+ def test_unreadable_docs_are_counted(self):
1848
+ # A corrupt (bad-CRC) spine doc reads as empty text; the completeness
1849
+ # record must surface it as an unreadable fraction, never as prose.
1850
+ import struct
1851
+ import zipfile as zf
1852
+
1853
+ docs = [(f"ch{i}.xhtml", self._prose(7, f"Chapter {i}")) for i in range(1, 4)]
1854
+ docs.append(("broken.xhtml", self._prose(7, "Chapter 4")))
1855
+ with tempfile.TemporaryDirectory() as tmp:
1856
+ p = self._epub(tmp, docs)
1857
+ with zf.ZipFile(p) as z:
1858
+ offset = z.getinfo("broken.xhtml").header_offset
1859
+ with open(p, "r+b") as f:
1860
+ f.seek(offset + 26)
1861
+ nlen, elen = struct.unpack("<HH", f.read(4))
1862
+ f.seek(offset + 30 + nlen + elen + 10)
1863
+ byte = f.read(1)
1864
+ f.seek(-1, os.SEEK_CUR)
1865
+ f.write(bytes([byte[0] ^ 0xFF]))
1866
+ r = audit.analyze_completeness(audit.load_book(p))
1867
+ self.assertEqual(r["spine_docs"], 4)
1868
+ self.assertEqual(r["unreadable"], 1)
1869
+ self.assertGreater(r["unreadable_frac"], 0.0)
1870
+ self.assertEqual(r["prose_docs"], 3)
1871
+
1872
+
1873
+ class TestTrailingTocClassifier(unittest.TestCase):
1874
+ """The trailing-ToC shape test directly: a chapter list is dense with
1875
+ short link lines; any doc with real paragraphs is not a ToC."""
1876
+
1877
+ def test_link_list_is_toc(self):
1878
+ body = "<h1>Contents</h1>" + "".join(
1879
+ f'<p><a href="ch{i}.xhtml">Chapter {i}: An Entry With Some Length</a></p>'
1880
+ for i in range(1, 9)
1881
+ )
1882
+ self.assertTrue(audit._trailing_toc(body))
1883
+
1884
+ def test_prose_doc_is_not_a_toc(self):
1885
+ body = "<h2>Chapter One</h2><p>" + ("Ordinary prose runs here. " * 40) + "</p>"
1886
+ self.assertFalse(audit._trailing_toc(body))
1887
+
1888
+ def test_heading_followed_by_prose_is_not_a_toc(self):
1889
+ # the Mattimeo guard: a heading line followed by real paragraphs is
1890
+ # content, even when other headings crowd the document
1891
+ body = "".join(
1892
+ f"<h2>Chapter {i}</h2><p>" + ("Ordinary prose runs here. " * 30) + "</p>"
1893
+ for i in range(1, 6)
1894
+ )
1895
+ self.assertFalse(audit._trailing_toc(body))
1896
+
1897
+ def test_short_link_run_is_not_a_toc(self):
1898
+ # a colophon or a prev/next pager: too few lines to call a ToC
1899
+ body = "".join(
1900
+ f'<p><a href="ch{i}.xhtml">Chapter {i}</a></p>' for i in range(1, 4)
1901
+ )
1902
+ self.assertFalse(audit._trailing_toc(body))
1903
+
1904
+ def test_prose_without_links_is_not_a_toc(self):
1905
+ # a dialogue-heavy doc of many short paragraphs carries no links
1906
+ body = "".join(f"<p>Said the hare, line {i}.</p>" for i in range(1, 21))
1907
+ self.assertFalse(audit._trailing_toc(body))
1908
+
1909
+
1910
+ class TestCompletenessVerdict(unittest.TestCase):
1911
+ """The completeness verdict is advisory by contract: the archive, spine,
1912
+ and emptytext verdicts own the flags; completeness reports the shape."""
1913
+
1914
+ def _r(self, **over):
1915
+ r = {
1916
+ "spine_docs": 31,
1917
+ "spine_missing": 0,
1918
+ "prose_docs": 30,
1919
+ "unreadable": 0,
1920
+ "unreadable_frac": 0.0,
1921
+ "blank_docs": 0,
1922
+ "trailing_toc": "",
1923
+ "spots": [
1924
+ {
1925
+ "doc": "ch01.xhtml",
1926
+ "chars": 5000,
1927
+ "opens": "It began",
1928
+ "ends": "it ended",
1929
+ }
1930
+ ],
1931
+ }
1932
+ r.update(over)
1933
+ return r
1934
+
1935
+ def test_clean_book_is_ok_and_never_a_problem(self):
1936
+ problem, status, lines = audit._completeness_dir(self._r())
1937
+ self.assertFalse(problem)
1938
+ self.assertEqual(status, "OK")
1939
+ self.assertTrue(any("prose 30/31" in ln for ln in lines))
1940
+ self.assertTrue(any("ch01.xhtml" in ln for ln in lines))
1941
+
1942
+ def test_trailing_toc_is_advisory(self):
1943
+ problem, status, lines = audit._completeness_dir(
1944
+ self._r(trailing_toc="toc.xhtml", prose_docs=30)
1945
+ )
1946
+ self.assertFalse(problem)
1947
+ self.assertEqual(status, "ADVISORY")
1948
+ self.assertTrue(any("toc.xhtml" in ln for ln in lines))
1949
+
1950
+ def test_high_unreadable_fraction_is_advisory(self):
1951
+ problem, status, _ = audit._completeness_dir(
1952
+ self._r(unreadable=6, unreadable_frac=0.12)
1953
+ )
1954
+ self.assertFalse(problem)
1955
+ self.assertEqual(status, "ADVISORY")
1956
+
1957
+ def test_blank_docs_are_reported(self):
1958
+ _problem, _status, lines = audit._completeness_dir(self._r(blank_docs=2))
1959
+ self.assertTrue(any("blank" in ln for ln in lines))
1960
+
1961
+
1962
+ class TestCompletenessInAll(unittest.TestCase):
1963
+ """`all` runs the completeness analyzer in the same single decompression
1964
+ pass, and the --json payload carries its verdict per book."""
1965
+
1966
+ def test_all_includes_completeness(self):
1967
+ self.assertIn("completeness", audit.ALL)
1968
+
1969
+ def test_directory_json_record_carries_completeness(self):
1970
+ import zipfile as zf
1971
+
1972
+ para = "The hares marched along the ridge in the fine morning light. "
1973
+ body = "<h2>Chapter One</h2><p>" + (para * 8) + "</p>"
1974
+ with tempfile.TemporaryDirectory() as tmp:
1975
+ p = pathlib.Path(tmp) / "t.epub"
1976
+ with zf.ZipFile(p, "w") as z:
1977
+ z.writestr("mimetype", "application/epub+zip")
1978
+ z.writestr("META-INF/container.xml", TestEmptyTextScan.CONTAINER)
1979
+ z.writestr("content.opf", TestEmptyTextScan.OPF)
1980
+ z.writestr("text.xhtml", f"<html><body>{body}</body></html>")
1981
+ out = pathlib.Path(tmp) / "audit.json"
1982
+ buf = io.StringIO()
1983
+ with contextlib.redirect_stdout(buf):
1984
+ rc = audit.run_directory(
1985
+ pathlib.Path(tmp),
1986
+ ["completeness"],
1987
+ 2000,
1988
+ 20000,
1989
+ json_path=out,
1990
+ )
1991
+ payload = json.loads(out.read_text())
1992
+ self.assertEqual(rc, 0) # advisory: never fails the run
1993
+ self.assertIn("completeness", payload["analyzers"])
1994
+ rec = payload["books"][0]
1995
+ self.assertIn("completeness", rec["verdicts"])
1996
+ self.assertEqual(rec["verdicts"]["completeness"]["status"], "OK")
1997
+ self.assertIn("prose 1/1", rec["verdicts"]["completeness"]["details"][0])
@@ -19,7 +19,7 @@ reserialize = [
19
19
 
20
20
  [package.metadata]
21
21
  requires-dist = [
22
- { name = "cquarry", specifier = ">=1.17.0" },
22
+ { name = "cquarry", specifier = ">=1.19.0" },
23
23
  { name = "html5lib", marker = "extra == 'reserialize'", specifier = ">=1.1" },
24
24
  { name = "tqdm" },
25
25
  { name = "vir-tui", specifier = ">=2.2.0" },
@@ -37,11 +37,11 @@ wheels = [
37
37
 
38
38
  [[package]]
39
39
  name = "cquarry"
40
- version = "1.17.0"
40
+ version = "1.19.0"
41
41
  source = { registry = "https://pypi.org/simple" }
42
- sdist = { url = "https://files.pythonhosted.org/packages/4d/6b/e2c80511bf62e3b008c52d47a05b29e43e2782614cf211aadc8c965a2be1/cquarry-1.17.0.tar.gz", hash = "sha256:07ec6c43ebc11f8921645a963373537ef4808985c88ad885de5dfc05ba08d737", size = 197288, upload-time = "2026-09-09T21:48:07.208Z" }
42
+ sdist = { url = "https://files.pythonhosted.org/packages/d5/a1/7bb89bf5095c2dc64dc07029a540fb46af863fce1e7f042a479c25c46625/cquarry-1.19.0.tar.gz", hash = "sha256:34c87bd3c73957f9b32a54bd3fc6746820b40b003d333fffa2f678a288e403aa", size = 241248, upload-time = "2026-09-12T16:43:11.865Z" }
43
43
  wheels = [
44
- { url = "https://files.pythonhosted.org/packages/4d/5f/90f6fe1c24a3ce8616cb377cbedee3dfb82c4eca84cbb1ec8b02a7034cb1/cquarry-1.17.0-py3-none-any.whl", hash = "sha256:f01c5b4ecb0863c78379e9fc899fd0da7d2518e815ed2a9000acec09cbea054b", size = 70998, upload-time = "2026-09-09T21:48:05.798Z" },
44
+ { url = "https://files.pythonhosted.org/packages/2b/a7/36392bbfa1bebea7fad521ef01d93291df54b1343a2e9efd60eedd04556d/cquarry-1.19.0-py3-none-any.whl", hash = "sha256:7b0d417ac43218e1605db8c2458172f4ba688d8b12b3872a6e7ba03842fdcb98", size = 84386, upload-time = "2026-09-12T16:43:10.145Z" },
45
45
  ]
46
46
 
47
47
  [[package]]
File without changes
File without changes
File without changes
File without changes
File without changes