bindery-cli 0.35.0__tar.gz → 0.36.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/CLAUDE.md +3 -3
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/PKG-INFO +10 -5
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/README.md +8 -3
- bindery_cli-0.36.0/REPORT-12-Sept.md +119 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/patchnotes.md +52 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/pyproject.toml +2 -2
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/roadmap.md +100 -3
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/spec.md +16 -4
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/__init__.py +1 -1
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/audit.py +183 -5
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/cli.py +13 -4
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_audit.py +283 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/uv.lock +4 -4
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/.github/workflows/ci.yml +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/.github/workflows/publish.yml +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/.gitignore +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/AGENTS.md +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/LICENSE +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/docs/screenshots/library-sweep.png +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/logo.svg +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/run_tests.sh +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/scripts/FastSweep.java +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/scripts/fast_sweep.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/__main__.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/epub.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/library.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/pagination.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/reserialize.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/transforms.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/validate.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/src/bindery/watermark.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/testing_facility/books_to_fix/.gitkeep +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/testing_facility/top500candidates/REPORT.md +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_cli.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_epub.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_library.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_pagination.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_reserialize.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_transforms.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_validate.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_version.py +0 -0
- {bindery_cli-0.35.0 → bindery_cli-0.36.0}/tests/test_watermark.py +0 -0
|
@@ -77,12 +77,12 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
|
|
|
77
77
|
- `src/bindery/pagination.py`: the opt-in lossy page-number strip (runhead detection, page-layer decision, block-centric removal/merge, safety nets).
|
|
78
78
|
- `src/bindery/watermark.py`: the opt-in lossy watermark strip (anchored and anchorless signature removal).
|
|
79
79
|
- `src/bindery/reserialize.py`: structural repair via `html5lib`.
|
|
80
|
-
- `src/bindery/audit.py`: read-only body-text audits (`content`, `pagenumbers`, `emptytext`, `ocr`,
|
|
80
|
+
- `src/bindery/audit.py`: read-only body-text audits (`content`, `pagenumbers`, `emptytext`, `ocr`, `monolithic` since v0.21.0 with `--max-doc-chars N`, and `completeness` since v0.36.0) behind the `audit` subcommand (v0.15.0). Since v0.18.0 library mode resolves EPUB paths through `cquarry.get_format_path()` and can apply a tag to flagged books via `cquarry.write.WritableCalibreDB` (only with explicit `--tag`; the only sanctioned write path). Since v0.19.0 `audit --id BOOK_ID` audits a single library book through cquarry's single-entity `get_book()` fetch (no library-wide cache; supports `--tag`; incompatible with the directory argument; comma lists since v0.23.0). Since v0.22.0 every archive entry is fully read (CRC + decompression) before analysis — a damaged entry reports its own CORRUPT verdict instead of feeding emptytext — and manifest/NCX references to absent files classify as `convention` (bloated ToC, consecutive span) or `fragment` (broken span). Hazard: the scan loop reuses `tag` as its per-book display column, so the `--tag` argument is captured as `audit_tag` at function entry — do not collapse them again. Since v0.29.0 `audit --json FILE` writes per-file analyzer verdicts in the `library --json` shape (directory, library, and single-book modes; with `--id`, exactly one id; emptytext omitted from a record when the archive verdict owns the body-text story). Since v0.36.0 the completeness analyzer (Phase 15) reports the phase-1 spot-check per book: spine/prose-doc counts (prose = >= 400 visible chars), first/middle/last prose-doc opening/closing excerpts, trailing-ToC classification (`_trailing_toc`: short link lines dominate, no paragraph-length blocks), and the unreadable fraction. It is ADVISORY by contract (problem always False; ADVISORY on a trailing ToC or unreadable fraction >= 10%), never moves the exit code, and is skipped like emptytext when the archive verdict owns the book.
|
|
81
81
|
- `src/bindery/epub.py`: archive rewrite, NCX uid sync, RepairReport, mismatch detection, and the opt-in structural-repair plumbing (including the CSS precondition scan).
|
|
82
82
|
- `src/bindery/validate.py`: epubcheck wrapper, the `gate` (improvement) and `no_worse` (no-regression, for the lossy strips) acceptance bars.
|
|
83
|
-
- `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching)
|
|
83
|
+
- `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching); the `(id)` directory-name regex is only the no-catalog fallback. Since v0.24.0 `install_format()` places the repaired file atomically and updates the `data` row through `cquarry.write.WritableCalibreDB` (since v0.31.0 via `set_format`, cquarry 1.17's sanctioned remove+add in one transaction); the external `calibredb` CLI is gone. Since v0.32.0 a guessed id drives a row update only when metadata.db corroborates it (`_verify_guess`: books row exists, file in the book's own directory, catalogued `data.name` when a row exists); anything else saves in place and leaves the catalog untouched — never reintroduce directory-name guessing as the primary source: renamed/mismatched directories would replace the wrong book.
|
|
84
84
|
- `src/bindery/cli.py`: `repair` and `library` subcommands, including `--all` and `--install-to-calibre`; since v0.29.0 also the `run` verbs: `phase1` (pre-import vetting over loose files: the audit battery composed with one gated repair sweep; read-only until `--apply-lossy`, which IS the lossy-strip consent) and `phase3` (the scoped post-import apply step: `library --id --sweep --only all --apply --all --install-to-calibre` plus a pre/post summary; mechanically refuses unscoped sweeps with exit 2). Both take `--non-interactive` and surface open questions as `decisions_needed` in their JSON, never prompts. They drive the shipped `run_library` through the real parser, so no flag can drift between the verb and the subcommand it wraps. `library --workers N` (v0.30.0) parallelizes only the sweep's candidate pass, in windows of N consumed in order so `--limit` stays lazy; the repair phase is never parallel (shared workdir, atomic-replacement contract).
|
|
85
|
-
- `tests/`: transforms, end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic, archive integrity, spine classification), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, and the version pin.
|
|
85
|
+
- `tests/`: transforms, end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic/completeness, archive integrity, spine classification), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, and the version pin.
|
|
86
86
|
|
|
87
87
|
## Conventions
|
|
88
88
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bindery-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.36.0
|
|
4
4
|
Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
|
|
5
5
|
Author: vrnvctss
|
|
6
6
|
License: MIT
|
|
@@ -13,7 +13,7 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.14
|
|
14
14
|
Classifier: Topic :: Utilities
|
|
15
15
|
Requires-Python: >=3.14
|
|
16
|
-
Requires-Dist: cquarry>=1.
|
|
16
|
+
Requires-Dist: cquarry>=1.19.0
|
|
17
17
|
Requires-Dist: tqdm
|
|
18
18
|
Requires-Dist: vir-tui>=2.2.0
|
|
19
19
|
Provides-Extra: reserialize
|
|
@@ -57,8 +57,8 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
|
|
|
57
57
|
- **`--unwrap-block-in-inline`**: safely unwraps `<span>` tags that illegally contain a block-level element (e.g. `<div>` or `<p>`), leaving the block element intact.
|
|
58
58
|
- **`--strip-invalid-value`**: systematically strips invalid `value="..."` attributes from elements like `<div>`, `<span>`, `<p>`, etc.
|
|
59
59
|
- **`--fix-page-map`**: normalizes legacy page-map markup — drops the non-standard `page-map="..."` attribute from the OPF `<spine>` and adds `class="pages"` to classless NCX `<pageList>` elements (epubcheck rejects both on older HarperCollins / Anna's Archive conversions).
|
|
60
|
-
- **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package
|
|
61
|
-
- **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents
|
|
60
|
+
- **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package: `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive). The edit is anchored to real start tags, so prose that merely mentions `epub:type="chapter"` is preserved.
|
|
61
|
+
- **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents (`figure`/`section` to `div`, `figcaption` to `p`), keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed. Both EPUB2-targeted fixes are gated on the package version declared in the OPF: on an EPUB 3 book they do nothing, because their target defects only exist below EPUB 3.
|
|
62
62
|
- **`--unwrap-illegal-tags`**: strips completely invalid or deprecated HTML tags that break EPUB3 validation (`<st>`, `<sentence>`, `<o>`, `<w>`, `<pagebreak>`) while retaining their inner text. Any of those names styled as an *element selector* by an EPUB stylesheet (`.css` entries and inline `<style>` blocks alike; class/id selectors like `.st`/`#w` don't count) is protected for the whole book, guaranteeing format preservation.
|
|
63
63
|
- **`--prune-missing-resources`**: removes references to files the archive does not contain (RSC-007/PKG-010): dead `<link>` elements, anchors' `href` to absent files (anchor text preserved), absent `<img>` sources (replaced by their alt text when they carry one), and orphaned non-spine OPF manifest items. Spine documents are never pruned: a missing spine document is reported as a damaged fragment, not silently dropped.
|
|
64
64
|
- **`--strip-broken-anchors`**: strips href attributes that cannot resolve, keeping the anchor text byte-for-byte: a `#fragment` the target document does not define (RSC-020/RSC-012; NCX navTargets fall back to the document target, so chapter navigation survives) and unresolvable URI schemes (`kindle:`, `file:`).
|
|
@@ -126,10 +126,15 @@ bindery audit ocr ~/docs/Calibre\ Library
|
|
|
126
126
|
# Flag books where a single content document is too large for readers to render
|
|
127
127
|
bindery audit monolithic ~/docs/Calibre\ Library
|
|
128
128
|
|
|
129
|
+
# Spot-check completeness: prose coverage, first/middle/last excerpts, trailing ToC
|
|
130
|
+
bindery audit completeness ~/docs/Calibre\ Library
|
|
131
|
+
|
|
129
132
|
# Run all audits and generate a comprehensive CSV
|
|
130
133
|
cd ~/docs/Calibre\ Library && bindery audit all
|
|
131
134
|
```
|
|
132
135
|
|
|
136
|
+
**Completeness spot-check:** the `completeness` audit answers the pre-import question "does this book reach its end?" It reports, per book: the prose-document count against the spine (docs with 400+ visible characters), the opening and closing text of the first/middle/last prose doc, a trailing-ToC classification of the final spine doc (a link-dense chapter list after the real back matter), and the fraction of unreadable docs. A trailing ToC is reported as an advisory, never a failure: the closing excerpt always comes from the real back matter, so a human can confirm the book ends where the author ended it.
|
|
137
|
+
|
|
133
138
|
Audits can also be run on a directory of loose `.epub` files by passing the path as the second argument:
|
|
134
139
|
```sh
|
|
135
140
|
bindery audit pagenumbers /path/to/loose/epubs
|
|
@@ -204,7 +209,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
|
|
|
204
209
|
- `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
|
|
205
210
|
- `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
|
|
206
211
|
- `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
|
|
207
|
-
- `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied
|
|
212
|
+
- `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied`; Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning. A directory-name-guessed id is trusted only when `metadata.db` corroborates it (the book exists and the file is that book's catalogued EPUB); an uncatalogued stray file inside a book directory is saved in place and the catalog row is left untouched.
|
|
208
213
|
- `--all` automatically turns on all opt-in non-fatal fixes and lossy strips (pagination, watermarks, bad attributes, unknown entities, image alt tags, etc.) in a single run.
|
|
209
214
|
- Only the `.epub` is replaced. `metadata.opf`, `cover.jpg`, and `metadata.db` are left for Calibre's Quality Check sync to reconcile.
|
|
210
215
|
- A per-book progress line goes to stderr (stdout stays a clean report); `--quiet` suppresses it. A corrupt or unreadable book is reported and skipped, never aborting the sweep.
|
|
@@ -35,8 +35,8 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
|
|
|
35
35
|
- **`--unwrap-block-in-inline`**: safely unwraps `<span>` tags that illegally contain a block-level element (e.g. `<div>` or `<p>`), leaving the block element intact.
|
|
36
36
|
- **`--strip-invalid-value`**: systematically strips invalid `value="..."` attributes from elements like `<div>`, `<span>`, `<p>`, etc.
|
|
37
37
|
- **`--fix-page-map`**: normalizes legacy page-map markup — drops the non-standard `page-map="..."` attribute from the OPF `<spine>` and adds `class="pages"` to classless NCX `<pageList>` elements (epubcheck rejects both on older HarperCollins / Anna's Archive conversions).
|
|
38
|
-
- **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package
|
|
39
|
-
- **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents
|
|
38
|
+
- **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package: `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive). The edit is anchored to real start tags, so prose that merely mentions `epub:type="chapter"` is preserved.
|
|
39
|
+
- **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents (`figure`/`section` to `div`, `figcaption` to `p`), keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed. Both EPUB2-targeted fixes are gated on the package version declared in the OPF: on an EPUB 3 book they do nothing, because their target defects only exist below EPUB 3.
|
|
40
40
|
- **`--unwrap-illegal-tags`**: strips completely invalid or deprecated HTML tags that break EPUB3 validation (`<st>`, `<sentence>`, `<o>`, `<w>`, `<pagebreak>`) while retaining their inner text. Any of those names styled as an *element selector* by an EPUB stylesheet (`.css` entries and inline `<style>` blocks alike; class/id selectors like `.st`/`#w` don't count) is protected for the whole book, guaranteeing format preservation.
|
|
41
41
|
- **`--prune-missing-resources`**: removes references to files the archive does not contain (RSC-007/PKG-010): dead `<link>` elements, anchors' `href` to absent files (anchor text preserved), absent `<img>` sources (replaced by their alt text when they carry one), and orphaned non-spine OPF manifest items. Spine documents are never pruned: a missing spine document is reported as a damaged fragment, not silently dropped.
|
|
42
42
|
- **`--strip-broken-anchors`**: strips href attributes that cannot resolve, keeping the anchor text byte-for-byte: a `#fragment` the target document does not define (RSC-020/RSC-012; NCX navTargets fall back to the document target, so chapter navigation survives) and unresolvable URI schemes (`kindle:`, `file:`).
|
|
@@ -104,10 +104,15 @@ bindery audit ocr ~/docs/Calibre\ Library
|
|
|
104
104
|
# Flag books where a single content document is too large for readers to render
|
|
105
105
|
bindery audit monolithic ~/docs/Calibre\ Library
|
|
106
106
|
|
|
107
|
+
# Spot-check completeness: prose coverage, first/middle/last excerpts, trailing ToC
|
|
108
|
+
bindery audit completeness ~/docs/Calibre\ Library
|
|
109
|
+
|
|
107
110
|
# Run all audits and generate a comprehensive CSV
|
|
108
111
|
cd ~/docs/Calibre\ Library && bindery audit all
|
|
109
112
|
```
|
|
110
113
|
|
|
114
|
+
**Completeness spot-check:** the `completeness` audit answers the pre-import question "does this book reach its end?" It reports, per book: the prose-document count against the spine (docs with 400+ visible characters), the opening and closing text of the first/middle/last prose doc, a trailing-ToC classification of the final spine doc (a link-dense chapter list after the real back matter), and the fraction of unreadable docs. A trailing ToC is reported as an advisory, never a failure: the closing excerpt always comes from the real back matter, so a human can confirm the book ends where the author ended it.
|
|
115
|
+
|
|
111
116
|
Audits can also be run on a directory of loose `.epub` files by passing the path as the second argument:
|
|
112
117
|
```sh
|
|
113
118
|
bindery audit pagenumbers /path/to/loose/epubs
|
|
@@ -182,7 +187,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
|
|
|
182
187
|
- `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
|
|
183
188
|
- `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
|
|
184
189
|
- `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
|
|
185
|
-
- `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied
|
|
190
|
+
- `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied`; Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning. A directory-name-guessed id is trusted only when `metadata.db` corroborates it (the book exists and the file is that book's catalogued EPUB); an uncatalogued stray file inside a book directory is saved in place and the catalog row is left untouched.
|
|
186
191
|
- `--all` automatically turns on all opt-in non-fatal fixes and lossy strips (pagination, watermarks, bad attributes, unknown entities, image alt tags, etc.) in a single run.
|
|
187
192
|
- Only the `.epub` is replaced. `metadata.opf`, `cover.jpg`, and `metadata.db` are left for Calibre's Quality Check sync to reconcile.
|
|
188
193
|
- A per-book progress line goes to stderr (stdout stays a clean report); `--quiet` suppresses it. A corrupt or unreadable book is reported and skipped, never aborting the sweep.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# REPORT-12-Sept: the repair-landscape research (bindery-cli vs real-world EPUB damage)
|
|
2
|
+
|
|
3
|
+
Research date: 2026-09-10/11. Method: a read-only research pass mapping
|
|
4
|
+
the full taxonomy of real-world EPUB damage — by producer (Calibre
|
|
5
|
+
conversions, Word/Office export, InDesign, legacy OCR, Adobe Digital
|
|
6
|
+
Editions, KindleUnpack, self-pub aggregators, Windows case-manglers) —
|
|
7
|
+
against this repo's shipped repair surface, the Phase 14/15 backlogs,
|
|
8
|
+
and the Bug Reports. References include the upstream clone's format
|
|
9
|
+
readers. Full evidence lives in the agent's findings; this report is
|
|
10
|
+
the distilled verdict and every actionable gap with its routing.
|
|
11
|
+
|
|
12
|
+
## Verdict
|
|
13
|
+
|
|
14
|
+
Bindery handles roughly **60-65% of real-world damage classes
|
|
15
|
+
end-to-end** and is near-complete for its founding niche: markup
|
|
16
|
+
well-formedness (the conversion-fatals class), NCX basics,
|
|
17
|
+
zip/mimetype container hygiene, dead-reference pruning, watermark and
|
|
18
|
+
page-number furniture, and archive/spine diagnosis (corruption, DRM
|
|
19
|
+
skip, ToC-bloat classification). Phase 14's hardening and all five Bug
|
|
20
|
+
Reports are shipped through v0.35.0; Phase 15 (the completeness
|
|
21
|
+
analyzer) is approved.
|
|
22
|
+
|
|
23
|
+
The missing ~35% is not more markup surgery. It is **package-level
|
|
24
|
+
repairs** — container.xml, manifest media-types, spine attribute
|
|
25
|
+
cleanup, encryption.xml/obfuscation state, zip-level dedupe, href
|
|
26
|
+
case/IRI mismatches — and the finding that matters most: several of
|
|
27
|
+
those classes are **gateway defects**. Epubcheck stays fatal while
|
|
28
|
+
they persist, so no shipped repair can ever be gate-accepted on those
|
|
29
|
+
books. They are locked out of the pipeline entirely.
|
|
30
|
+
|
|
31
|
+
## A. The damage-taxonomy matrix and the repair gaps
|
|
32
|
+
|
|
33
|
+
1. **Missing/corrupt `META-INF/container.xml`** (any producer):
|
|
34
|
+
epubcheck fatal; the OPF is unfindable. The locator falls back to
|
|
35
|
+
the first `.opf` but never writes the file. Deterministic fix:
|
|
36
|
+
generate the standard container.xml at the located OPF (upstream
|
|
37
|
+
calibre's `initialize_container` is the model). Effort S; HIGH
|
|
38
|
+
value — a gateway repair that unlocks an entire class of books.
|
|
39
|
+
2. **Stale font-obfuscation state** (publisher EPUB3 fonts; IDPF
|
|
40
|
+
`2008/embedding` and Adobe `pdf/enc#RC`): post-KindleUnpack/strip
|
|
41
|
+
tools decrypt fonts in place but leave `encryption.xml` declaring
|
|
42
|
+
obfuscation; epubcheck errors and the audit implies DRM where there
|
|
43
|
+
is none. Fix: detect via font magic bytes at offset 0 vs the
|
|
44
|
+
CipherReference; prune stale entries; drop encryption.xml when only
|
|
45
|
+
obfuscation entries remain and the fonts verify. Never
|
|
46
|
+
un-obfuscate. Also gives the audit a real OBFUSCATED-vs-DRM
|
|
47
|
+
distinction. Effort M; needs the magic-byte detection validated on
|
|
48
|
+
real samples first.
|
|
49
|
+
3. **Wrong/missing manifest `media-type`** (aggregators, ADE): fonts,
|
|
50
|
+
images, and media declared with wrong types. Fix: sniff magic bytes
|
|
51
|
+
or extension map; attribute-only. Effort S.
|
|
52
|
+
4. **href case/backslash/IRI mismatches** (KindleUnpack, Windows
|
|
53
|
+
producers): `Images/Cover.JPG` vs `images/cover.jpg`, backslash
|
|
54
|
+
separators, `%XX` variants — the file IS present, so
|
|
55
|
+
`--prune-missing-resources` cannot touch the RSC-007/PKG-010 it
|
|
56
|
+
causes. Fix: case-insensitive namemap when exactly one match;
|
|
57
|
+
backslash to slash. Effort M.
|
|
58
|
+
5. **Dangling reference edges after pruning** (any producer):
|
|
59
|
+
`spine@toc`, `item@media-overlay`, `item@fallback`, and EPUB2 cover
|
|
60
|
+
meta pointing at items the prune pass deleted. Today's prune checks
|
|
61
|
+
only `spine_ids` — it can manufacture the regression the gate then
|
|
62
|
+
rejects. Fix: extend prune to rewrite those edges (the rewrite
|
|
63
|
+
machinery exists in fix_manifest_ids). Effort S/M.
|
|
64
|
+
6. **Duplicate zip entries** (any producer): last-wins shadowing; the
|
|
65
|
+
audit counts `dup_entries`, the rewrite copies both. Fix: drop
|
|
66
|
+
shadowed duplicates on rewrite. Effort S.
|
|
67
|
+
7. **Cover wiring** (EPUB2 producers): dangling `<meta name="cover">`,
|
|
68
|
+
missing EPUB3 `properties="cover-image"`, wrong cover media-type —
|
|
69
|
+
calibre and booksellers extract no cover. Narrow, finding-driven
|
|
70
|
+
repair when the cover is identifiable from the existing guide/meta;
|
|
71
|
+
audit-only otherwise. Effort M.
|
|
72
|
+
8. **Broken CSS from converters**: `url()` to absent assets, malformed
|
|
73
|
+
selectors. Repair is fiddly; ship the audit-only detector first and
|
|
74
|
+
let prevalence data decide. Effort M (repair).
|
|
75
|
+
9. **Human-facing metadata validity** (self-pub/Word): non-ISO
|
|
76
|
+
`dc:date`, invalid `dc:language`, missing EPUB3
|
|
77
|
+
`dcterms:modified`. Explicit non-goal today; `dcterms:modified`
|
|
78
|
+
also collides with byte-determinism. Needs a written charter
|
|
79
|
+
ruling: carve-out with a determinism-safe timestamp policy, or
|
|
80
|
+
route to cquarry/CalibreQuarry. Effort M + a decision.
|
|
81
|
+
10. **EPUB3 dual-ToC drift** (nav vs NCX): detection is a
|
|
82
|
+
deterministic structural diff feeding `decisions_needed`; full
|
|
83
|
+
sync is ToC synthesis and stays out of charter permanently.
|
|
84
|
+
Effort L if repair is ever attempted; audit-only recommended.
|
|
85
|
+
11. **No-repair classes, correctly routed**: OCR garbage, truncated
|
|
86
|
+
re-sources, absent font families (audit + re-source); real DRM
|
|
87
|
+
(clean ENCRYPTED skip — shipped); DB-level format rows
|
|
88
|
+
(cquarry's, done); catalog completeness and human-facing metadata
|
|
89
|
+
quality (CalibreQuarry's audits).
|
|
90
|
+
|
|
91
|
+
## B. Research-before-boxing
|
|
92
|
+
|
|
93
|
+
1. **FastSweep prevalence study** — run `fast_sweep.py --mode=extract`
|
|
94
|
+
aggregation over the full library targeting the new classes
|
|
95
|
+
(container.xml presence/validity, encryption.xml algorithms,
|
|
96
|
+
media-type warnings, case-mismatch RSC-007s). The harness exists;
|
|
97
|
+
it sizes every box above with real counts before any is built.
|
|
98
|
+
2. **The metadata carve-out ruling** (gap 9) — Brandon decides;
|
|
99
|
+
the audit records both options.
|
|
100
|
+
3. **The obfuscation magic-byte table** (gap 2) — validate IDPF and
|
|
101
|
+
Adobe font prefixes against real samples before the repair ships.
|
|
102
|
+
|
|
103
|
+
## C. Routing notes
|
|
104
|
+
|
|
105
|
+
Package-structure repairs (items 1-6) fit the existing "structural
|
|
106
|
+
opt-ins + gate" model without reopening the semantics-preserving
|
|
107
|
+
contract. Items 7-10 each need an explicit scope decision first.
|
|
108
|
+
Cross-repo: DB-level size/format-row mismatches route to cquarry
|
|
109
|
+
(set_format, shipped); catalog completeness, duplicate detection, and
|
|
110
|
+
human-facing metadata quality route to CalibreQuarry's audits; OCR,
|
|
111
|
+
truncation, monolithic, and missing-font content stay audit-and-
|
|
112
|
+
re-source permanently.
|
|
113
|
+
|
|
114
|
+
## D. Phase 16 routing
|
|
115
|
+
|
|
116
|
+
Items 1-6 (the ungated package-structure repairs) are boxed in
|
|
117
|
+
roadmap.md **Phase 16** with the FastSweep prevalence study leading.
|
|
118
|
+
Items 7-10 are recorded as decision-gated boxes. Phase 15 (approved)
|
|
119
|
+
runs first if its fixture work is not yet done.
|
|
@@ -1,4 +1,56 @@
|
|
|
1
1
|
# bindery-cli Patch Notes
|
|
2
|
+
## v0.36.0 (2026-09-12)
|
|
3
|
+
|
|
4
|
+
### The completeness spot-check: `audit completeness` (Phase 15)
|
|
5
|
+
|
|
6
|
+
- **The phase-1 judgment step finally has a tool owner.** The import
|
|
7
|
+
skill's completeness step asks a human question with no tool behind
|
|
8
|
+
it: "sample early, middle, and late pages ... read the LAST content
|
|
9
|
+
page to confirm it reaches real back matter rather than cutting off
|
|
10
|
+
mid-chapter". Two consecutive phase-1 runs answered it with a
|
|
11
|
+
hand-rolled zipfile sampler in /tmp. The new analyzer rides the
|
|
12
|
+
audit's existing single decompression pass (no second decode of any
|
|
13
|
+
book) and reports, per book: spine doc count, prose-doc count (spine
|
|
14
|
+
docs with at least 400 visible characters), the opening and closing
|
|
15
|
+
120 characters of the first/middle/last prose doc, a trailing-ToC
|
|
16
|
+
classification of the final spine doc, and the fraction of unreadable
|
|
17
|
+
docs (corrupt entries plus unresolved itemrefs).
|
|
18
|
+
- **A trailing ToC never impersonates the ending.** The final spine doc
|
|
19
|
+
is classified as a chapter list when short link lines dominate it and
|
|
20
|
+
no block runs to paragraph length; such a doc is book furniture and
|
|
21
|
+
is excluded from the prose sampling, so the closing excerpt comes
|
|
22
|
+
from the real back matter. Verified against the real 2026-09-10
|
|
23
|
+
Redwall fixtures: Lord Brocktree's trailing ToC is caught and the
|
|
24
|
+
Epilogue is sampled as the last prose doc; The Long Patrol reads
|
|
25
|
+
exactly 55/57 prose docs through the percent-encoded hrefs that
|
|
26
|
+
defeated the hand-rolled sampler; Mattimeo's split doc (chapter
|
|
27
|
+
headings with real prose after every one) stays prose.
|
|
28
|
+
- **Advisory by contract.** The analyzer never flags a book and never
|
|
29
|
+
moves the exit code: the archive, spine, and emptytext verdicts keep
|
|
30
|
+
owning the flags. The verdict status is ADVISORY (not OK) when a
|
|
31
|
+
trailing ToC is present or the unreadable fraction reaches 10%.
|
|
32
|
+
`audit all` and `run phase1` include it through the battery, the CLI
|
|
33
|
+
grows the `completeness` mode, and `--json` records carry its
|
|
34
|
+
verdicts in the same shape as the other analyzers.
|
|
35
|
+
|
|
36
|
+
### Dependencies
|
|
37
|
+
|
|
38
|
+
- Adopted cquarry 1.18.0 (floor + lock): the set_format repair lane now keeps
|
|
39
|
+
Calibre's FTS index and page counts honest automatically (the sidecar's
|
|
40
|
+
dirty queue and the pages rescan flag ride the write), and the search and
|
|
41
|
+
runtime fixes come along in the shared layer.
|
|
42
|
+
- Adopted cquarry 1.19.0 (floor + lock): the repair lane gains its undo
|
|
43
|
+
primitive -- save_original_format/restore_original_format -- so a format
|
|
44
|
+
repair can put the original bytes back if it goes wrong.
|
|
45
|
+
|
|
46
|
+
### Repo records
|
|
47
|
+
|
|
48
|
+
- The release-tag backlog is settled forward-only from v0.35.0: the
|
|
49
|
+
nine untagged 0.18.0-0.26.0 entries are exempted by decision of
|
|
50
|
+
record (2026-09-11), and the v0.28.0 retag and the v0.33.0 em-dash
|
|
51
|
+
tag repair stay recorded-only force-pushes (roadmap.md, "Tag
|
|
52
|
+
policy").
|
|
53
|
+
|
|
2
54
|
## v0.35.0 (2026-09-10)
|
|
3
55
|
|
|
4
56
|
### The epubcheck daemon, reconciled: oracle-identical counts at ~15x the speed
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "bindery-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.36.0"
|
|
8
8
|
description = "Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
classifiers = [
|
|
@@ -27,7 +27,7 @@ authors = [
|
|
|
27
27
|
dependencies = [
|
|
28
28
|
"tqdm",
|
|
29
29
|
"vir-tui>=2.2.0",
|
|
30
|
-
"cquarry>=1.
|
|
30
|
+
"cquarry>=1.19.0",
|
|
31
31
|
]
|
|
32
32
|
|
|
33
33
|
[project.optional-dependencies]
|
|
@@ -663,7 +663,7 @@ book instead raises an uncaught `TypeError` at the `book["path"]` access.
|
|
|
663
663
|
matches the DB's book path and format name before the batch; otherwise save
|
|
664
664
|
in place and warn without touching the row, and catch the `TypeError` path.
|
|
665
665
|
*(Fixed in v0.32.0: a guessed id now drives a row update only when
|
|
666
|
-
metadata.db corroborates it
|
|
666
|
+
metadata.db corroborates it: the books row exists, the file sits in that
|
|
667
667
|
book's catalogued directory, and when an EPUB row exists it carries that
|
|
668
668
|
row's stored name. Any other guess (a stray file in a book directory, a
|
|
669
669
|
stale `(N)` directory whose book is gone, the resolver's id gone from
|
|
@@ -996,7 +996,7 @@ can be net-neutral and ships silently.*
|
|
|
996
996
|
v0.22.0 "its own verdict in every mode" promise (`audit.py:1565`,
|
|
997
997
|
`1783-1803`, `1520-1558`).
|
|
998
998
|
*(Done in v0.33.0: the archive and spine sections run unconditionally
|
|
999
|
-
after the selected content sections
|
|
999
|
+
after the selected content sections; they are always-on verdicts and
|
|
1000
1000
|
were already scanned and recorded in the JSON; only the console/rc
|
|
1001
1001
|
path was dead. A CRC-corrupt book now prints its CORRUPT section and
|
|
1002
1002
|
fails library mode with exit 1, matching directory mode. Test pins
|
|
@@ -1286,7 +1286,7 @@ can be net-neutral and ships silently.*
|
|
|
1286
1286
|
|
|
1287
1287
|
### Documentation (drift is real but narrow)
|
|
1288
1288
|
|
|
1289
|
-
- [
|
|
1289
|
+
- [x] **Reconcile spec.md with the shipped opt-ins.** The contract's
|
|
1290
1290
|
non-goals forbid exactly what `--reserialize` (v0.3.0, the lone
|
|
1291
1291
|
html5lib dependency) and `--strip-bad-attrs` (v0.4.0) do, and neither
|
|
1292
1292
|
has a spec section; the OPF half of `--fix-ids` is missing;
|
|
@@ -1294,6 +1294,10 @@ can be net-neutral and ships silently.*
|
|
|
1294
1294
|
NCX sentence at spec.md:52-53 is garbled (a dangling "to the OPF
|
|
1295
1295
|
unique identifier" tail). A contributor reading spec.md first would
|
|
1296
1296
|
conclude two shipped flags violate the charter.
|
|
1297
|
+
*(APPLIED 2026-09-10, commit 01ad173, on Brandon's verbatim go
|
|
1298
|
+
recorded in the audit sheet's evening answers; shipped with
|
|
1299
|
+
v0.34.0. Ticked 2026-09-11 in the main-thread double-check after
|
|
1300
|
+
the verifier caught the work shipping unticked.)*
|
|
1297
1301
|
**GATED on Brandon: spec.md is the contract file. Draft proposal,
|
|
1298
1302
|
ready to apply on a go (do not apply without one):**
|
|
1299
1303
|
(1) Non-goals bullet 2 ("Repairing genuinely mangled structure
|
|
@@ -1368,6 +1372,14 @@ everything else is backlog, not danger.*
|
|
|
1368
1372
|
|
|
1369
1373
|
## Phase 15: completeness spot-check analyzer (proposed 2026-09-10, from the phase-1 skill's own doctrine)
|
|
1370
1374
|
|
|
1375
|
+
*(SHIPPED 2026-09-12 as v0.36.0: `audit completeness` rides the existing
|
|
1376
|
+
single decompression pass; the Redwall-shaped regression tests landed
|
|
1377
|
+
first (4021ce7), the analyzer second (e6156c0). Verified against the
|
|
1378
|
+
real fixtures: Lord Brocktree's trailing ToC is caught with the Epilogue
|
|
1379
|
+
sampled as the last prose doc, The Long Patrol reads exactly 55/57
|
|
1380
|
+
prose docs through its percent-encoded hrefs, and Mattimeo's split doc
|
|
1381
|
+
stays prose. Advisory by contract; `all` and `run phase1` include it.)*
|
|
1382
|
+
|
|
1371
1383
|
*(RAISED with Brandon 2026-09-10, batched with the session-start raises;
|
|
1372
1384
|
recommendation: APPROVE. Execution is gated on that go; the proposal
|
|
1373
1385
|
below is filed and costed. The Redwall run it names is a ready-made
|
|
@@ -1392,3 +1404,88 @@ Brocktree (real Epilogue doc before a trailing 520-char ToC), Mattimeo
|
|
|
1392
1404
|
(Chapter 50 verified as a real heading with prose after, inside a 174k-char
|
|
1393
1405
|
split doc), The Long Patrol (percent-encoded hrefs defeated the hand-rolled
|
|
1394
1406
|
reader; 55/57 prose docs).*
|
|
1407
|
+
|
|
1408
|
+
## Phase 16: package-structure repairs (proposed 2026-09-12, from REPORT-12-Sept.md)
|
|
1409
|
+
|
|
1410
|
+
A research pass mapped the full real-world EPUB damage taxonomy against
|
|
1411
|
+
the shipped repair surface: bindery handles ~60-65% of damage classes
|
|
1412
|
+
end-to-end, and the missing slice is package-level structure — several
|
|
1413
|
+
classes of which are GATEWAY DEFECTS (epubcheck stays fatal, so no
|
|
1414
|
+
shipped repair can ever be gate-accepted on those books). Full evidence,
|
|
1415
|
+
producer classes, and the cross-repo routing live in
|
|
1416
|
+
REPORT-12-Sept.md. Lead with the prevalence study; every box below is
|
|
1417
|
+
an opt-in structural repair under the existing gate contract.
|
|
1418
|
+
|
|
1419
|
+
### A. Research first (boxes cannot be honestly sized without these)
|
|
1420
|
+
|
|
1421
|
+
- [ ] **FastSweep prevalence study**: run `fast_sweep.py
|
|
1422
|
+
--mode=extract` aggregation over the full library for the new
|
|
1423
|
+
classes — container.xml presence/validity, encryption.xml
|
|
1424
|
+
algorithms, manifest media-type warnings, case-mismatch RSC-007s.
|
|
1425
|
+
Each Phase 16 repair box gets its real count before it opens.
|
|
1426
|
+
- [ ] **Obfuscation magic-byte validation**: confirm the IDPF
|
|
1427
|
+
(`2008/embedding`) and Adobe (`pdf/enc#RC`) font prefixes at offset 0
|
|
1428
|
+
against real samples before the encryption-repair box opens.
|
|
1429
|
+
|
|
1430
|
+
### B. The repairs (each: opt-in flag, dry-run default, gate-accepted)
|
|
1431
|
+
|
|
1432
|
+
- [ ] **`--fix-container`**: generate META-INF/container.xml at the
|
|
1433
|
+
located OPF (model: upstream calibre's initialize_container).
|
|
1434
|
+
Gateway repair, effort S, HIGH value.
|
|
1435
|
+
- [ ] **Obfuscation-aware encryption.xml repair**: prune stale
|
|
1436
|
+
obfuscation entries when font magic bytes prove the fonts are
|
|
1437
|
+
decrypted in place; drop encryption.xml when only obfuscation entries
|
|
1438
|
+
remain and fonts verify. NEVER un-obfuscate. M.
|
|
1439
|
+
- [ ] **Audit distinction: OBFUSCATED vs DRM** in audit.py (today the
|
|
1440
|
+
ENCRYPTED verdict keys on unreadable entries and implies DRM where
|
|
1441
|
+
publisher obfuscation is benign). S, rides the box above.
|
|
1442
|
+
- [ ] **Reference-edge completion for `--prune-missing-resources`**:
|
|
1443
|
+
rewrite `spine@toc`, `item@media-overlay`, `item@fallback`, and EPUB2
|
|
1444
|
+
cover-meta edges when their targets are pruned (machinery exists in
|
|
1445
|
+
fix_manifest_ids). Prevents manufacturing gate regressions. S/M.
|
|
1446
|
+
- [ ] **Manifest media-type normalization**: sniff magic bytes /
|
|
1447
|
+
extension map; attribute-only. S.
|
|
1448
|
+
- [ ] **href case/backslash resolution**: case-insensitive namemap when
|
|
1449
|
+
exactly one zip entry matches; backslash to slash. Kills the
|
|
1450
|
+
present-file RSC-007/PKG-010 slice prune cannot touch. M.
|
|
1451
|
+
- [ ] **Duplicate zip-entry dedupe on rewrite**: drop shadowed
|
|
1452
|
+
duplicates (audit already counts `dup_entries`). S.
|
|
1453
|
+
|
|
1454
|
+
### C. Decision-gated (recorded boxes; no work without a ruling)
|
|
1455
|
+
|
|
1456
|
+
- [ ] **Cover wiring repair** (dangling `<meta name="cover">`, missing
|
|
1457
|
+
`properties="cover-image"`, wrong cover media-type): narrow,
|
|
1458
|
+
finding-driven repair vs audit-only. Needs the scope decision. M.
|
|
1459
|
+
- [ ] **Metadata validity carve-out ruling** (non-ISO `dc:date`,
|
|
1460
|
+
invalid `dc:language`, missing `dcterms:modified`): bindery opt-in
|
|
1461
|
+
with a determinism-safe timestamp policy, or route to
|
|
1462
|
+
cquarry/CalibreQuarry. Needs a written charter ruling. M.
|
|
1463
|
+
- [ ] **NCX<->nav drift DETECTOR** (audit-only; structural diff feeding
|
|
1464
|
+
`decisions_needed`; ToC synthesis stays out of charter
|
|
1465
|
+
permanently). L if repair is ever attempted.
|
|
1466
|
+
- [ ] **CSS url() pruning** as `--prune-missing-resources` scope
|
|
1467
|
+
extension: only after the prevalence study says it matters. M.
|
|
1468
|
+
|
|
1469
|
+
Ship shape: A leads; then B in value order (container, encryption,
|
|
1470
|
+
prune-edges, media-types); C boxes record their decisions and wait.
|
|
1471
|
+
Each repair ships with a fixture built from the prevalence study's
|
|
1472
|
+
real samples, a regression test written before the fix, and the gate
|
|
1473
|
+
(epubcheck) accepting the result.
|
|
1474
|
+
|
|
1475
|
+
## Tag policy (decided 2026-09-11, recorded 2026-09-12)
|
|
1476
|
+
|
|
1477
|
+
The release-tag backlog is settled, forward-only:
|
|
1478
|
+
|
|
1479
|
+
- **v0.35.0 is the anchor.** Every release from v0.36.0 on is tagged at
|
|
1480
|
+
its release commit under auto-tag promotion (a fully qualified release
|
|
1481
|
+
is tagged and pushed without asking; the blitz rulebook's decision 22).
|
|
1482
|
+
- **The nine untagged 0.18.0-0.26.0 entries are exempted** by decision
|
|
1483
|
+
of record (2026-09-11): no backfill tags will be cut for them. Their
|
|
1484
|
+
release notes live in patchnotes.md and their wheels are on PyPI; the
|
|
1485
|
+
missing git tags are accepted history.
|
|
1486
|
+
- **Two recorded-only repairs stay force-pushes, undecided:** the
|
|
1487
|
+
v0.28.0 retag (the tag points at a red commit) and the v0.33.0
|
|
1488
|
+
em-dash tag message (the pushed tag carries two em-dashes the
|
|
1489
|
+
proofread missed; the source patchnotes entry was fixed immediately in
|
|
1490
|
+
1c374c1). Either would need publish.yml disabled around the push and
|
|
1491
|
+
Brandon's explicit go; neither is ever auto-executed.
|
|
@@ -243,8 +243,8 @@ daemon (`_EpubcheckDaemon`, a bounded pool sized by `--workers`) that writes
|
|
|
243
243
|
`FastDaemon.java` at runtime and launches it through Java's single-file source
|
|
244
244
|
launcher (in-memory compilation by the running JVM: no javac, no version skew),
|
|
245
245
|
driving the warm JVM over a pipe. The daemon answers with the counts it reads from
|
|
246
|
-
the JSON that epubcheck's own `CheckingReport.generate()` serializes
|
|
247
|
-
document `--json` produces
|
|
246
|
+
the JSON that epubcheck's own `CheckingReport.generate()` serializes (the same
|
|
247
|
+
document `--json` produces), so both paths measure identically by construction.
|
|
248
248
|
(Epubcheck's JSON counts aggregated messages, not occurrences: a defect repeated
|
|
249
249
|
three times is one error. The gate has always measured that scale.) Every daemon
|
|
250
250
|
roundtrip is bounded by the caller's timeout, and any daemon failure tears it down
|
|
@@ -343,8 +343,8 @@ crashing the sweep.
|
|
|
343
343
|
|
|
344
344
|
## Audit subcommand (read-only)
|
|
345
345
|
|
|
346
|
-
`bindery audit {content,pagenumbers,emptytext,ocr,monolithic,all} [PATH] [--max-doc-chars N]
|
|
347
|
-
[--tag TAG] [--id IDs]` (v0.15.0, `audit.py`; `--tag` since v0.18.0; `monolithic` since v0.21.0
|
|
346
|
+
`bindery audit {content,pagenumbers,emptytext,ocr,monolithic,completeness,all} [PATH] [--max-doc-chars N]
|
|
347
|
+
[--tag TAG] [--id IDs]` (v0.15.0, `audit.py`; `--tag` since v0.18.0; `monolithic` since v0.21.0, `--max-doc-chars N`; `--id` since v0.19.0, comma-lists in v0.23.0; `completeness` since v0.36.0) inspects
|
|
348
348
|
EPUB body text for flaws epubcheck cannot see: non-English script blocks, baked-in page-number
|
|
349
349
|
layers (sliding-window density heuristics), empty or thin books, systemic OCR damage, and
|
|
350
350
|
single oversized content documents (one spine doc at or above 300k characters — readers refuse
|
|
@@ -353,6 +353,18 @@ is fully read for CRC + decompression, reporting CORRUPT rather than EMPTY), and
|
|
|
353
353
|
issues. Manifest/NCX references to absent files are classified as either `convention` (ToC is bloated but
|
|
354
354
|
present documents form a consecutive chapter span) or `fragment` (the span itself is broken).
|
|
355
355
|
|
|
356
|
+
The completeness analyzer is the phase-1 spot-check and is advisory by contract: it never
|
|
357
|
+
flags a book and never moves the exit code. Per book it reports the spine doc count, the
|
|
358
|
+
prose-doc count (spine docs carrying at least 400 visible characters), the opening and closing
|
|
359
|
+
120 characters of the first/middle/last prose doc, a trailing-ToC classification of the final
|
|
360
|
+
spine doc, and the fraction of spine docs that could not be read (corrupt entries plus
|
|
361
|
+
unresolved itemrefs). A trailing ToC (the final doc is dominated by short link lines, at
|
|
362
|
+
least half its lines carry links, and no block runs to paragraph length) is book furniture:
|
|
363
|
+
it is excluded from the prose sampling so the closing excerpt comes from the real back
|
|
364
|
+
matter, and the verdict becomes ADVISORY (as it does at an unreadable fraction of 10% or
|
|
365
|
+
more). Like emptytext it is skipped wholesale when the archive verdict owns the body-text
|
|
366
|
+
story.
|
|
367
|
+
|
|
356
368
|
In library mode, EPUB files are resolved through `cquarry.db.CalibreDB.get_format_path()` — the
|
|
357
369
|
storage-layout logic is not duplicated here. The scan itself still writes nothing. The opt-in
|
|
358
370
|
`--tag TAG` pass applies `TAG` to flagged books via `cquarry.write.WritableCalibreDB`, the
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
"""
|
|
3
3
|
bindery audit: read the actual text of every EPUB and flag content problems that
|
|
4
|
-
metadata and structural validators cannot see.
|
|
4
|
+
metadata and structural validators cannot see. Six analyzers, one tool:
|
|
5
5
|
|
|
6
6
|
content non-English bodies (wrong-language editions) and injected
|
|
7
7
|
foreign-language ad-notices (declared lang=eng, body Portuguese
|
|
@@ -20,7 +20,14 @@ metadata and structural validators cannot see. Four analyzers, one tool:
|
|
|
20
20
|
their hyphenated form). Character-substitution errors ("sonic"
|
|
21
21
|
for "some") are OUT OF SCOPE: catching those needs a wordlist,
|
|
22
22
|
and this tool is stdlib-only by contract.
|
|
23
|
-
|
|
23
|
+
monolithic one spine doc so large that readers refuse to render it, though
|
|
24
|
+
the book totals normally
|
|
25
|
+
completeness the phase-1 spot-check: prose-doc count against the spine, the
|
|
26
|
+
opening/closing text of the first/middle/last prose doc, a
|
|
27
|
+
trailing-ToC classification of the final doc, and the fraction
|
|
28
|
+
of unreadable docs (advisory: it reports the shape a human
|
|
29
|
+
judges, it never flags the book)
|
|
30
|
+
all run all six in a SINGLE decompression pass per book
|
|
24
31
|
|
|
25
32
|
This merges the former audit_epub_content.py / audit_epub_pagenumbers.py /
|
|
26
33
|
audit_epub_emptytext.py: they shared the same spine resolution, library/
|
|
@@ -1127,6 +1134,132 @@ def scan_monolithic(path: Path) -> dict:
|
|
|
1127
1134
|
return analyze_monolithic(load_book(path))
|
|
1128
1135
|
|
|
1129
1136
|
|
|
1137
|
+
# ----------------------------------------------------------------------------
|
|
1138
|
+
# Analyzer: completeness (the phase-1 spot-check: does the book reach its end?)
|
|
1139
|
+
# ----------------------------------------------------------------------------
|
|
1140
|
+
|
|
1141
|
+
# A spine doc at or above this many visible chars is real content; under it is
|
|
1142
|
+
# front/back matter (title pages, colophons) or furniture.
|
|
1143
|
+
PROSE_DOC_MIN_CHARS = 400
|
|
1144
|
+
SPOT_CHARS = 120 # opening/closing excerpt length per sampled doc
|
|
1145
|
+
# A book this far unreadable (corrupt entries + unresolved itemrefs against
|
|
1146
|
+
# all spine itemrefs) draws the advisory, though it never fails a run.
|
|
1147
|
+
COMPLETENESS_UNREADABLE_FLAG = 0.10
|
|
1148
|
+
|
|
1149
|
+
# Trailing-ToC detection, tuned on the 2026-09-10 Redwall run: Lord
|
|
1150
|
+
# Brocktree's trailing ToC is a 520-char link list sitting AFTER a real
|
|
1151
|
+
# Epilogue doc, while Mattimeo's final doc carries chapter headings with real
|
|
1152
|
+
# prose after every one of them and must never classify as a ToC. A ToC page
|
|
1153
|
+
# is dense with short link lines and carries no real paragraphs.
|
|
1154
|
+
TRAILING_TOC_MIN_BLOCKS = 4 # fewer lines is a colophon or pager, not a ToC
|
|
1155
|
+
TRAILING_TOC_SHORT_MAX = 100 # a chapter-heading line is short
|
|
1156
|
+
TRAILING_TOC_SHORT_FRAC = 0.75 # most lines are short
|
|
1157
|
+
TRAILING_TOC_LONG_BLOCK = 400 # a real paragraph runs longer than any ToC line
|
|
1158
|
+
TRAILING_TOC_LINK_FRAC = 0.5 # at least half the lines carry a link
|
|
1159
|
+
|
|
1160
|
+
_TOC_BLOCK_SPLIT_RE = re.compile(r"</(?:p|div|li|h[1-6]|td)>", re.IGNORECASE)
|
|
1161
|
+
_A_TAG_RE = re.compile(r"<a[\s>]", re.IGNORECASE)
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
def _trailing_toc(html_src: str) -> bool:
|
|
1165
|
+
"""Is this document a chapter list? Short link lines dominate and no
|
|
1166
|
+
block runs to paragraph length; a heading followed by real prose (the
|
|
1167
|
+
Mattimeo split-doc case) keeps a document out."""
|
|
1168
|
+
body = _SCRIPT_STYLE_RE.sub(" ", html_src)
|
|
1169
|
+
blocks: list[str] = []
|
|
1170
|
+
for chunk in _TOC_BLOCK_SPLIT_RE.split(body):
|
|
1171
|
+
text = _visible_text(chunk)
|
|
1172
|
+
if text:
|
|
1173
|
+
blocks.append(text)
|
|
1174
|
+
if len(blocks) < TRAILING_TOC_MIN_BLOCKS:
|
|
1175
|
+
return False
|
|
1176
|
+
short = sum(1 for b in blocks if len(b) <= TRAILING_TOC_SHORT_MAX)
|
|
1177
|
+
if short / len(blocks) < TRAILING_TOC_SHORT_FRAC:
|
|
1178
|
+
return False
|
|
1179
|
+
if any(len(b) > TRAILING_TOC_LONG_BLOCK for b in blocks):
|
|
1180
|
+
return False
|
|
1181
|
+
links = len(_A_TAG_RE.findall(body))
|
|
1182
|
+
return links >= len(blocks) * TRAILING_TOC_LINK_FRAC
|
|
1183
|
+
|
|
1184
|
+
|
|
1185
|
+
def analyze_completeness(book: Book) -> dict:
|
|
1186
|
+
"""The phase-1 completeness spot-check, over the (pre-read) spine.
|
|
1187
|
+
|
|
1188
|
+
Reports the shape a human needs in order to judge "does this book reach
|
|
1189
|
+
its end": the prose-doc count against the spine, opening/closing excerpts
|
|
1190
|
+
of the first/middle/last prose doc, a trailing-ToC classification of the
|
|
1191
|
+
final doc, and the fraction of spine docs that could not be read at all
|
|
1192
|
+
(corrupt entries plus unresolved itemrefs). A trailing ToC is book
|
|
1193
|
+
furniture: it is excluded from the prose sampling so the closing excerpt
|
|
1194
|
+
comes from the real back matter (Lord Brocktree's Epilogue, not its
|
|
1195
|
+
520-char ToC). Reuses the shared visible_texts() cache: no second pass.
|
|
1196
|
+
"""
|
|
1197
|
+
texts = book.visible_texts()
|
|
1198
|
+
spine_n = len(book.spine)
|
|
1199
|
+
trailing = ""
|
|
1200
|
+
if spine_n >= 2 and _trailing_toc(book.docs.get(book.spine[-1], "")):
|
|
1201
|
+
trailing = book.spine[-1]
|
|
1202
|
+
prose = [
|
|
1203
|
+
(book.spine[i], texts[i])
|
|
1204
|
+
for i in range(spine_n)
|
|
1205
|
+
if len(texts[i]) >= PROSE_DOC_MIN_CHARS and book.spine[i] != trailing
|
|
1206
|
+
]
|
|
1207
|
+
corrupt_set = set(book.corrupt)
|
|
1208
|
+
unreadable = (
|
|
1209
|
+
sum(1 for i in range(spine_n) if not texts[i] and book.spine[i] in corrupt_set)
|
|
1210
|
+
+ book.spine_missing
|
|
1211
|
+
)
|
|
1212
|
+
blank_docs = sum(
|
|
1213
|
+
1 for i in range(spine_n) if not texts[i] and book.spine[i] not in corrupt_set
|
|
1214
|
+
)
|
|
1215
|
+
spots: list[dict] = []
|
|
1216
|
+
if prose:
|
|
1217
|
+
picks = sorted({0, len(prose) // 2, len(prose) - 1})
|
|
1218
|
+
spots = [
|
|
1219
|
+
{
|
|
1220
|
+
"doc": prose[i][0],
|
|
1221
|
+
"chars": len(prose[i][1]),
|
|
1222
|
+
"opens": prose[i][1][:SPOT_CHARS],
|
|
1223
|
+
"ends": prose[i][1][-SPOT_CHARS:],
|
|
1224
|
+
}
|
|
1225
|
+
for i in picks
|
|
1226
|
+
]
|
|
1227
|
+
total = spine_n + book.spine_missing
|
|
1228
|
+
return {
|
|
1229
|
+
"spine_docs": spine_n,
|
|
1230
|
+
"spine_missing": book.spine_missing,
|
|
1231
|
+
"prose_docs": len(prose),
|
|
1232
|
+
"unreadable": unreadable,
|
|
1233
|
+
"unreadable_frac": (unreadable / total) if total else 0.0,
|
|
1234
|
+
"blank_docs": blank_docs,
|
|
1235
|
+
"trailing_toc": trailing,
|
|
1236
|
+
"spots": spots,
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
def scan_completeness(path: Path) -> dict:
|
|
1241
|
+
return analyze_completeness(load_book(path))
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
def _completeness_dir(r: dict) -> tuple[bool, str, list[str]]:
|
|
1245
|
+
"""Advisory by contract: the archive, spine, and emptytext verdicts own
|
|
1246
|
+
the flags; completeness reports the shape a human judges."""
|
|
1247
|
+
lines = [
|
|
1248
|
+
f"prose {r['prose_docs']}/{r['spine_docs']} spine docs, "
|
|
1249
|
+
f"unreadable {r['unreadable']} ({r['unreadable_frac'] * 100:.0f}%)"
|
|
1250
|
+
]
|
|
1251
|
+
status = "OK"
|
|
1252
|
+
if r["trailing_toc"]:
|
|
1253
|
+
lines.append(f"trailing ToC: {r['trailing_toc']}")
|
|
1254
|
+
status = "ADVISORY"
|
|
1255
|
+
elif r["unreadable_frac"] >= COMPLETENESS_UNREADABLE_FLAG:
|
|
1256
|
+
status = "ADVISORY"
|
|
1257
|
+
if r["blank_docs"]:
|
|
1258
|
+
lines.append(f"{r['blank_docs']} spine doc(s) present but blank")
|
|
1259
|
+
lines += [f"{s['doc']}: [{s['opens']}] ... [{s['ends']}]" for s in r["spots"]]
|
|
1260
|
+
return (False, status, lines)
|
|
1261
|
+
|
|
1262
|
+
|
|
1130
1263
|
# ----------------------------------------------------------------------------
|
|
1131
1264
|
# Analyzer: archive corruption (always on; owns the "empty body" story)
|
|
1132
1265
|
# ----------------------------------------------------------------------------
|
|
@@ -1682,11 +1815,40 @@ def _spine_sections(hits, advisory) -> int:
|
|
|
1682
1815
|
return 0
|
|
1683
1816
|
|
|
1684
1817
|
|
|
1818
|
+
def _completeness_sections(advisory) -> int:
|
|
1819
|
+
"""Print the completeness advisories; the analyzer never fails a run."""
|
|
1820
|
+
if advisory:
|
|
1821
|
+
print(
|
|
1822
|
+
f"{YELLOW}{BOLD}COMPLETENESS SPOT-CHECK ({len(advisory)} to eyeball;"
|
|
1823
|
+
f" reported, not flagged){RESET}"
|
|
1824
|
+
)
|
|
1825
|
+
for book_id, title, tag, r in sorted(advisory):
|
|
1826
|
+
print(f" {YELLOW}#{book_id}{RESET} [{tag}] {title}")
|
|
1827
|
+
_p, _s, lines = _completeness_dir(r)
|
|
1828
|
+
for ln in lines:
|
|
1829
|
+
print(f" {ln}")
|
|
1830
|
+
print()
|
|
1831
|
+
print(
|
|
1832
|
+
f"{YELLOW}{BOLD}completeness DONE{RESET}: read the excerpts above;"
|
|
1833
|
+
f" a trailing ToC or unreadable docs never fail the run."
|
|
1834
|
+
)
|
|
1835
|
+
return 0
|
|
1836
|
+
print(f"{GREEN}{BOLD}completeness CLEAN{RESET}: every book reads whole.")
|
|
1837
|
+
return 0
|
|
1838
|
+
|
|
1839
|
+
|
|
1685
1840
|
# ----------------------------------------------------------------------------
|
|
1686
1841
|
# Runner
|
|
1687
1842
|
# ----------------------------------------------------------------------------
|
|
1688
1843
|
|
|
1689
|
-
ALL: tuple[str, ...] = (
|
|
1844
|
+
ALL: tuple[str, ...] = (
|
|
1845
|
+
"content",
|
|
1846
|
+
"pagenumbers",
|
|
1847
|
+
"emptytext",
|
|
1848
|
+
"ocr",
|
|
1849
|
+
"monolithic",
|
|
1850
|
+
"completeness",
|
|
1851
|
+
)
|
|
1690
1852
|
|
|
1691
1853
|
|
|
1692
1854
|
def _record_verdict(
|
|
@@ -1813,6 +1975,7 @@ def run_library(
|
|
|
1813
1975
|
thin_hits: list[tuple] = []
|
|
1814
1976
|
ocr_found: list[tuple] = []
|
|
1815
1977
|
mono_hits: list[tuple] = []
|
|
1978
|
+
completeness_advisory: list[tuple] = []
|
|
1816
1979
|
corrupt_hits: list[tuple] = []
|
|
1817
1980
|
spine_hits: list[tuple] = []
|
|
1818
1981
|
spine_advisory: list[tuple] = []
|
|
@@ -1908,6 +2071,15 @@ def run_library(
|
|
|
1908
2071
|
mono_hits.append((book_id, title, tag, r))
|
|
1909
2072
|
_record_verdict(record, "monolithic", _monolithic_dir(r, max_doc_chars))
|
|
1910
2073
|
|
|
2074
|
+
if "completeness" in selected and not corrupt_r["n"]:
|
|
2075
|
+
r = analyze_completeness(book)
|
|
2076
|
+
_p, status, lines = _completeness_dir(r)
|
|
2077
|
+
if status != "OK":
|
|
2078
|
+
completeness_advisory.append((book_id, title, tag, r))
|
|
2079
|
+
# advisory by contract: problem is always False, so the record
|
|
2080
|
+
# status and the exit code are never moved by this analyzer
|
|
2081
|
+
_record_verdict(record, "completeness", (False, status, lines))
|
|
2082
|
+
|
|
1911
2083
|
print(f"Scanned {scanned} EPUBs in {library_root}\n")
|
|
1912
2084
|
rc = 0
|
|
1913
2085
|
multi = len(selected) > 1
|
|
@@ -1924,6 +2096,8 @@ def run_library(
|
|
|
1924
2096
|
rc |= _empty_sections(empty_hits, partial_hits, thin_hits)
|
|
1925
2097
|
elif key == "monolithic":
|
|
1926
2098
|
rc |= _monolithic_sections(mono_hits)
|
|
2099
|
+
elif key == "completeness":
|
|
2100
|
+
rc |= _completeness_sections(completeness_advisory)
|
|
1927
2101
|
else:
|
|
1928
2102
|
rc |= _ocr_sections(ocr_found)
|
|
1929
2103
|
if multi:
|
|
@@ -2104,7 +2278,7 @@ def run_directory(
|
|
|
2104
2278
|
for key in ALL:
|
|
2105
2279
|
if key not in selected:
|
|
2106
2280
|
continue
|
|
2107
|
-
if key
|
|
2281
|
+
if key in ("emptytext", "completeness") and corrupt_r["n"]:
|
|
2108
2282
|
continue # the archive verdict owns this book's body-text story
|
|
2109
2283
|
if key == "content":
|
|
2110
2284
|
problem, status, lines = _content_dir(analyze_content(book))
|
|
@@ -2118,6 +2292,8 @@ def run_directory(
|
|
|
2118
2292
|
problem, status, lines = _monolithic_dir(
|
|
2119
2293
|
analyze_monolithic(book), max_doc_chars
|
|
2120
2294
|
)
|
|
2295
|
+
elif key == "completeness":
|
|
2296
|
+
problem, status, lines = _completeness_dir(analyze_completeness(book))
|
|
2121
2297
|
else:
|
|
2122
2298
|
problem, status, lines = _ocr_dir(analyze_ocr(book))
|
|
2123
2299
|
if problem:
|
|
@@ -2272,7 +2448,7 @@ def run_single(
|
|
|
2272
2448
|
for key in ALL:
|
|
2273
2449
|
if key not in selected:
|
|
2274
2450
|
continue
|
|
2275
|
-
if key
|
|
2451
|
+
if key in ("emptytext", "completeness") and corrupt_r["n"]:
|
|
2276
2452
|
# Corruption owns the body-text story for this book; reporting
|
|
2277
2453
|
# EMPTY here would be the wrong disease.
|
|
2278
2454
|
continue
|
|
@@ -2288,6 +2464,8 @@ def run_single(
|
|
|
2288
2464
|
problem, status, lines = _monolithic_dir(
|
|
2289
2465
|
analyze_monolithic(book), max_doc_chars
|
|
2290
2466
|
)
|
|
2467
|
+
elif key == "completeness":
|
|
2468
|
+
problem, status, lines = _completeness_dir(analyze_completeness(book))
|
|
2291
2469
|
else:
|
|
2292
2470
|
problem, status, lines = _ocr_dir(analyze_ocr(book))
|
|
2293
2471
|
if problem:
|
|
@@ -808,8 +808,9 @@ def _phase1_status(audit_rec: dict | None, repair_rec: dict | None) -> str:
|
|
|
808
808
|
|
|
809
809
|
def _run_phase1_audit(root: Path) -> tuple[list[dict], int]:
|
|
810
810
|
"""Stage 1: the read-only audit battery (corruption sweep, content
|
|
811
|
-
battery, monolithic) exactly as
|
|
812
|
-
|
|
811
|
+
battery, monolithic, completeness spot-check) exactly as
|
|
812
|
+
`bindery audit all DIR` runs it, with the report captured through its
|
|
813
|
+
own --json payload in a temp file."""
|
|
813
814
|
with tempfile.TemporaryDirectory() as td:
|
|
814
815
|
out = Path(td) / "audit.json"
|
|
815
816
|
rc = run_directory(
|
|
@@ -900,7 +901,7 @@ def run_phase1(args) -> int:
|
|
|
900
901
|
print(f"Bindery run phase1 ({mode}): {root}\n")
|
|
901
902
|
|
|
902
903
|
print(
|
|
903
|
-
"== stage 1/2: audit battery (corruption sweep, content battery, monolithic) =="
|
|
904
|
+
"== stage 1/2: audit battery (corruption sweep, content battery, monolithic, completeness) =="
|
|
904
905
|
)
|
|
905
906
|
audit_records, audit_rc = _run_phase1_audit(root)
|
|
906
907
|
|
|
@@ -1357,7 +1358,15 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
1357
1358
|
)
|
|
1358
1359
|
audit.add_argument(
|
|
1359
1360
|
"mode",
|
|
1360
|
-
choices=(
|
|
1361
|
+
choices=(
|
|
1362
|
+
"content",
|
|
1363
|
+
"pagenumbers",
|
|
1364
|
+
"emptytext",
|
|
1365
|
+
"ocr",
|
|
1366
|
+
"monolithic",
|
|
1367
|
+
"completeness",
|
|
1368
|
+
"all",
|
|
1369
|
+
),
|
|
1361
1370
|
help="which audit to run",
|
|
1362
1371
|
)
|
|
1363
1372
|
audit.add_argument(
|
|
@@ -1712,3 +1712,286 @@ class TestLibraryTagEndToEnd(unittest.TestCase):
|
|
|
1712
1712
|
os.chdir(old)
|
|
1713
1713
|
self.assertEqual(rc, 1) # an EMPTY book is a finding
|
|
1714
1714
|
self.assertEqual(tagged, [(1,)])
|
|
1715
|
+
|
|
1716
|
+
|
|
1717
|
+
class TestCompletenessAnalyzer(unittest.TestCase):
|
|
1718
|
+
"""Phase 15: the completeness spot-check analyzer. The fixture shapes come
|
|
1719
|
+
from the 2026-09-10 Redwall phase-1 run: Lord Brocktree (a real Epilogue
|
|
1720
|
+
doc sitting before a trailing 520-char ToC), Mattimeo (a 174k-char split
|
|
1721
|
+
doc whose final chapter heading has real prose after it), and The Long
|
|
1722
|
+
Patrol (percent-encoded hrefs the hand-rolled sampler could not read at
|
|
1723
|
+
all; 55/57 prose docs). Synthetic fillers replicate each SHAPE; no real
|
|
1724
|
+
text lives in this suite."""
|
|
1725
|
+
|
|
1726
|
+
CONTAINER = TestEmptyTextScan.CONTAINER
|
|
1727
|
+
|
|
1728
|
+
PARA = "The hares marched along the ridge in the fine morning light. "
|
|
1729
|
+
|
|
1730
|
+
def _prose(self, paras=7, heading=None, closer=""):
|
|
1731
|
+
h = f"<h2>{heading}</h2>" if heading else ""
|
|
1732
|
+
return h + "<p>" + (self.PARA * paras) + "</p>" + closer
|
|
1733
|
+
|
|
1734
|
+
def _epub(self, tmp, docs, hrefs=None):
|
|
1735
|
+
# docs: list of (name, body_html); hrefs optionally remaps the
|
|
1736
|
+
# manifest href (percent-encoded forms) while the archive name stays
|
|
1737
|
+
import zipfile as zf
|
|
1738
|
+
|
|
1739
|
+
manifest = ""
|
|
1740
|
+
for i, (n, _) in enumerate(docs):
|
|
1741
|
+
href = (hrefs or {}).get(n, n)
|
|
1742
|
+
manifest += (
|
|
1743
|
+
f'<item id="d{i}" href="{href}" media-type="application/xhtml+xml"/>'
|
|
1744
|
+
)
|
|
1745
|
+
spine = "".join(f'<itemref idref="d{i}"/>' for i in range(len(docs)))
|
|
1746
|
+
opf = (
|
|
1747
|
+
'<package xmlns="http://www.idpf.org/2007/opf">'
|
|
1748
|
+
f"<manifest>{manifest}</manifest><spine>{spine}</spine></package>"
|
|
1749
|
+
)
|
|
1750
|
+
p = pathlib.Path(tmp) / "t.epub"
|
|
1751
|
+
with zf.ZipFile(p, "w") as z:
|
|
1752
|
+
z.writestr("mimetype", "application/epub+zip")
|
|
1753
|
+
z.writestr("META-INF/container.xml", self.CONTAINER)
|
|
1754
|
+
z.writestr("content.opf", opf)
|
|
1755
|
+
for n, body in docs:
|
|
1756
|
+
z.writestr(n, f"<html><body>{body}</body></html>")
|
|
1757
|
+
return p
|
|
1758
|
+
|
|
1759
|
+
def test_real_back_matter_before_trailing_toc(self):
|
|
1760
|
+
# Lord Brocktree shape: six real chapters and a real Epilogue doc,
|
|
1761
|
+
# then a link-dense ToC (over the prose threshold at 456 visible
|
|
1762
|
+
# chars, like the real 520-char one) as the FINAL spine doc. The ToC
|
|
1763
|
+
# must classify as trailing furniture: never a prose sample, so the
|
|
1764
|
+
# closing excerpt comes from the Epilogue's back matter.
|
|
1765
|
+
chapters = [
|
|
1766
|
+
(f"ch{i}.xhtml", self._prose(7, f"Chapter {i}")) for i in range(1, 7)
|
|
1767
|
+
]
|
|
1768
|
+
epilogue = (
|
|
1769
|
+
"epilogue.xhtml",
|
|
1770
|
+
self._prose(
|
|
1771
|
+
7,
|
|
1772
|
+
"Epilogue",
|
|
1773
|
+
closer="<p>And so the tale closes with the hares home at last.</p>",
|
|
1774
|
+
),
|
|
1775
|
+
)
|
|
1776
|
+
toc = (
|
|
1777
|
+
"toc.xhtml",
|
|
1778
|
+
"<h1>Contents</h1>"
|
|
1779
|
+
+ "".join(
|
|
1780
|
+
f'<p><a href="ch{i}.xhtml">Chapter {i}: The March of the '
|
|
1781
|
+
"Hares and the Siege of the Mountain</a></p>"
|
|
1782
|
+
for i in range(1, 7)
|
|
1783
|
+
)
|
|
1784
|
+
+ '<p><a href="epilogue.xhtml">Epilogue: Home at Salamandastron'
|
|
1785
|
+
"</a></p>",
|
|
1786
|
+
)
|
|
1787
|
+
docs = [("title.xhtml", "<p>A Tale of the Long Patrol</p>")]
|
|
1788
|
+
docs += chapters + [epilogue, toc]
|
|
1789
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
1790
|
+
r = audit.analyze_completeness(audit.load_book(self._epub(tmp, docs)))
|
|
1791
|
+
self.assertEqual(r["spine_docs"], 9)
|
|
1792
|
+
self.assertEqual(r["trailing_toc"], "toc.xhtml")
|
|
1793
|
+
# six chapters + the epilogue; the 456-char ToC is excluded
|
|
1794
|
+
self.assertEqual(r["prose_docs"], 7)
|
|
1795
|
+
self.assertEqual(r["spots"][-1]["doc"], "epilogue.xhtml")
|
|
1796
|
+
self.assertIn("home at last", r["spots"][-1]["ends"])
|
|
1797
|
+
|
|
1798
|
+
def test_final_split_doc_with_prose_after_heading_is_not_a_toc(self):
|
|
1799
|
+
# Mattimeo shape: one huge split doc carrying fifty chapter headings,
|
|
1800
|
+
# each followed by real prose (Chapter 50 was verified to have prose
|
|
1801
|
+
# after it). The link-line heuristics must never read it as a ToC,
|
|
1802
|
+
# and the closing excerpt must be the prose after the last heading.
|
|
1803
|
+
chapters = [
|
|
1804
|
+
(f"ch{i}.xhtml", self._prose(7, f"Chapter {i}")) for i in range(1, 6)
|
|
1805
|
+
]
|
|
1806
|
+
split_body = "".join(
|
|
1807
|
+
f"<h2>Chapter {i}</h2><p>{self.PARA * 4}</p>" for i in range(1, 51)
|
|
1808
|
+
)
|
|
1809
|
+
split = ("split.xhtml", split_body)
|
|
1810
|
+
docs = [("title.xhtml", "<p>A Tale of the Long Patrol</p>")]
|
|
1811
|
+
docs += chapters + [split]
|
|
1812
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
1813
|
+
r = audit.analyze_completeness(audit.load_book(self._epub(tmp, docs)))
|
|
1814
|
+
self.assertEqual(r["trailing_toc"], "")
|
|
1815
|
+
self.assertEqual(r["spine_docs"], 7)
|
|
1816
|
+
self.assertEqual(r["prose_docs"], 6) # five chapters + the split doc
|
|
1817
|
+
self.assertEqual(r["spots"][-1]["doc"], "split.xhtml")
|
|
1818
|
+
self.assertIn("fine morning light", r["spots"][-1]["ends"])
|
|
1819
|
+
|
|
1820
|
+
def test_percent_encoded_hrefs_resolve(self):
|
|
1821
|
+
# The Long Patrol shape: manifest hrefs carry %20 runs for the spaces
|
|
1822
|
+
# in the stored archive names (the hand-rolled sampler read none of
|
|
1823
|
+
# these docs). The shipped audit's IRI decoding resolves them for
|
|
1824
|
+
# free: every spine doc lands, short front matter stays out of the
|
|
1825
|
+
# prose set, and the final doc is sampled whole.
|
|
1826
|
+
docs = [
|
|
1827
|
+
("title.xhtml", "<p>A Tale of the Long Patrol</p>"),
|
|
1828
|
+
("note.xhtml", "<p>A brief historical note.</p>"),
|
|
1829
|
+
]
|
|
1830
|
+
docs += [
|
|
1831
|
+
(f"Chapter {i:02d}.xhtml", self._prose(7, f"Chapter {i}"))
|
|
1832
|
+
for i in range(1, 6)
|
|
1833
|
+
]
|
|
1834
|
+
docs.append(("The End.xhtml", self._prose(7, "Chapter Six")))
|
|
1835
|
+
hrefs = {n: n.replace(" ", "%20") for n, _ in docs}
|
|
1836
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
1837
|
+
r = audit.analyze_completeness(
|
|
1838
|
+
audit.load_book(self._epub(tmp, docs, hrefs))
|
|
1839
|
+
)
|
|
1840
|
+
self.assertEqual(r["spine_docs"], 8)
|
|
1841
|
+
self.assertEqual(r["prose_docs"], 6)
|
|
1842
|
+
self.assertEqual(r["trailing_toc"], "")
|
|
1843
|
+
self.assertEqual(r["spots"][-1]["doc"], "The End.xhtml")
|
|
1844
|
+
self.assertEqual(len(r["spots"]), 3)
|
|
1845
|
+
self.assertTrue(r["spots"][0]["opens"])
|
|
1846
|
+
|
|
1847
|
+
def test_unreadable_docs_are_counted(self):
|
|
1848
|
+
# A corrupt (bad-CRC) spine doc reads as empty text; the completeness
|
|
1849
|
+
# record must surface it as an unreadable fraction, never as prose.
|
|
1850
|
+
import struct
|
|
1851
|
+
import zipfile as zf
|
|
1852
|
+
|
|
1853
|
+
docs = [(f"ch{i}.xhtml", self._prose(7, f"Chapter {i}")) for i in range(1, 4)]
|
|
1854
|
+
docs.append(("broken.xhtml", self._prose(7, "Chapter 4")))
|
|
1855
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
1856
|
+
p = self._epub(tmp, docs)
|
|
1857
|
+
with zf.ZipFile(p) as z:
|
|
1858
|
+
offset = z.getinfo("broken.xhtml").header_offset
|
|
1859
|
+
with open(p, "r+b") as f:
|
|
1860
|
+
f.seek(offset + 26)
|
|
1861
|
+
nlen, elen = struct.unpack("<HH", f.read(4))
|
|
1862
|
+
f.seek(offset + 30 + nlen + elen + 10)
|
|
1863
|
+
byte = f.read(1)
|
|
1864
|
+
f.seek(-1, os.SEEK_CUR)
|
|
1865
|
+
f.write(bytes([byte[0] ^ 0xFF]))
|
|
1866
|
+
r = audit.analyze_completeness(audit.load_book(p))
|
|
1867
|
+
self.assertEqual(r["spine_docs"], 4)
|
|
1868
|
+
self.assertEqual(r["unreadable"], 1)
|
|
1869
|
+
self.assertGreater(r["unreadable_frac"], 0.0)
|
|
1870
|
+
self.assertEqual(r["prose_docs"], 3)
|
|
1871
|
+
|
|
1872
|
+
|
|
1873
|
+
class TestTrailingTocClassifier(unittest.TestCase):
|
|
1874
|
+
"""The trailing-ToC shape test directly: a chapter list is dense with
|
|
1875
|
+
short link lines; any doc with real paragraphs is not a ToC."""
|
|
1876
|
+
|
|
1877
|
+
def test_link_list_is_toc(self):
|
|
1878
|
+
body = "<h1>Contents</h1>" + "".join(
|
|
1879
|
+
f'<p><a href="ch{i}.xhtml">Chapter {i}: An Entry With Some Length</a></p>'
|
|
1880
|
+
for i in range(1, 9)
|
|
1881
|
+
)
|
|
1882
|
+
self.assertTrue(audit._trailing_toc(body))
|
|
1883
|
+
|
|
1884
|
+
def test_prose_doc_is_not_a_toc(self):
|
|
1885
|
+
body = "<h2>Chapter One</h2><p>" + ("Ordinary prose runs here. " * 40) + "</p>"
|
|
1886
|
+
self.assertFalse(audit._trailing_toc(body))
|
|
1887
|
+
|
|
1888
|
+
def test_heading_followed_by_prose_is_not_a_toc(self):
|
|
1889
|
+
# the Mattimeo guard: a heading line followed by real paragraphs is
|
|
1890
|
+
# content, even when other headings crowd the document
|
|
1891
|
+
body = "".join(
|
|
1892
|
+
f"<h2>Chapter {i}</h2><p>" + ("Ordinary prose runs here. " * 30) + "</p>"
|
|
1893
|
+
for i in range(1, 6)
|
|
1894
|
+
)
|
|
1895
|
+
self.assertFalse(audit._trailing_toc(body))
|
|
1896
|
+
|
|
1897
|
+
def test_short_link_run_is_not_a_toc(self):
|
|
1898
|
+
# a colophon or a prev/next pager: too few lines to call a ToC
|
|
1899
|
+
body = "".join(
|
|
1900
|
+
f'<p><a href="ch{i}.xhtml">Chapter {i}</a></p>' for i in range(1, 4)
|
|
1901
|
+
)
|
|
1902
|
+
self.assertFalse(audit._trailing_toc(body))
|
|
1903
|
+
|
|
1904
|
+
def test_prose_without_links_is_not_a_toc(self):
|
|
1905
|
+
# a dialogue-heavy doc of many short paragraphs carries no links
|
|
1906
|
+
body = "".join(f"<p>Said the hare, line {i}.</p>" for i in range(1, 21))
|
|
1907
|
+
self.assertFalse(audit._trailing_toc(body))
|
|
1908
|
+
|
|
1909
|
+
|
|
1910
|
+
class TestCompletenessVerdict(unittest.TestCase):
|
|
1911
|
+
"""The completeness verdict is advisory by contract: the archive, spine,
|
|
1912
|
+
and emptytext verdicts own the flags; completeness reports the shape."""
|
|
1913
|
+
|
|
1914
|
+
def _r(self, **over):
|
|
1915
|
+
r = {
|
|
1916
|
+
"spine_docs": 31,
|
|
1917
|
+
"spine_missing": 0,
|
|
1918
|
+
"prose_docs": 30,
|
|
1919
|
+
"unreadable": 0,
|
|
1920
|
+
"unreadable_frac": 0.0,
|
|
1921
|
+
"blank_docs": 0,
|
|
1922
|
+
"trailing_toc": "",
|
|
1923
|
+
"spots": [
|
|
1924
|
+
{
|
|
1925
|
+
"doc": "ch01.xhtml",
|
|
1926
|
+
"chars": 5000,
|
|
1927
|
+
"opens": "It began",
|
|
1928
|
+
"ends": "it ended",
|
|
1929
|
+
}
|
|
1930
|
+
],
|
|
1931
|
+
}
|
|
1932
|
+
r.update(over)
|
|
1933
|
+
return r
|
|
1934
|
+
|
|
1935
|
+
def test_clean_book_is_ok_and_never_a_problem(self):
|
|
1936
|
+
problem, status, lines = audit._completeness_dir(self._r())
|
|
1937
|
+
self.assertFalse(problem)
|
|
1938
|
+
self.assertEqual(status, "OK")
|
|
1939
|
+
self.assertTrue(any("prose 30/31" in ln for ln in lines))
|
|
1940
|
+
self.assertTrue(any("ch01.xhtml" in ln for ln in lines))
|
|
1941
|
+
|
|
1942
|
+
def test_trailing_toc_is_advisory(self):
|
|
1943
|
+
problem, status, lines = audit._completeness_dir(
|
|
1944
|
+
self._r(trailing_toc="toc.xhtml", prose_docs=30)
|
|
1945
|
+
)
|
|
1946
|
+
self.assertFalse(problem)
|
|
1947
|
+
self.assertEqual(status, "ADVISORY")
|
|
1948
|
+
self.assertTrue(any("toc.xhtml" in ln for ln in lines))
|
|
1949
|
+
|
|
1950
|
+
def test_high_unreadable_fraction_is_advisory(self):
|
|
1951
|
+
problem, status, _ = audit._completeness_dir(
|
|
1952
|
+
self._r(unreadable=6, unreadable_frac=0.12)
|
|
1953
|
+
)
|
|
1954
|
+
self.assertFalse(problem)
|
|
1955
|
+
self.assertEqual(status, "ADVISORY")
|
|
1956
|
+
|
|
1957
|
+
def test_blank_docs_are_reported(self):
|
|
1958
|
+
_problem, _status, lines = audit._completeness_dir(self._r(blank_docs=2))
|
|
1959
|
+
self.assertTrue(any("blank" in ln for ln in lines))
|
|
1960
|
+
|
|
1961
|
+
|
|
1962
|
+
class TestCompletenessInAll(unittest.TestCase):
|
|
1963
|
+
"""`all` runs the completeness analyzer in the same single decompression
|
|
1964
|
+
pass, and the --json payload carries its verdict per book."""
|
|
1965
|
+
|
|
1966
|
+
def test_all_includes_completeness(self):
|
|
1967
|
+
self.assertIn("completeness", audit.ALL)
|
|
1968
|
+
|
|
1969
|
+
def test_directory_json_record_carries_completeness(self):
|
|
1970
|
+
import zipfile as zf
|
|
1971
|
+
|
|
1972
|
+
para = "The hares marched along the ridge in the fine morning light. "
|
|
1973
|
+
body = "<h2>Chapter One</h2><p>" + (para * 8) + "</p>"
|
|
1974
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
1975
|
+
p = pathlib.Path(tmp) / "t.epub"
|
|
1976
|
+
with zf.ZipFile(p, "w") as z:
|
|
1977
|
+
z.writestr("mimetype", "application/epub+zip")
|
|
1978
|
+
z.writestr("META-INF/container.xml", TestEmptyTextScan.CONTAINER)
|
|
1979
|
+
z.writestr("content.opf", TestEmptyTextScan.OPF)
|
|
1980
|
+
z.writestr("text.xhtml", f"<html><body>{body}</body></html>")
|
|
1981
|
+
out = pathlib.Path(tmp) / "audit.json"
|
|
1982
|
+
buf = io.StringIO()
|
|
1983
|
+
with contextlib.redirect_stdout(buf):
|
|
1984
|
+
rc = audit.run_directory(
|
|
1985
|
+
pathlib.Path(tmp),
|
|
1986
|
+
["completeness"],
|
|
1987
|
+
2000,
|
|
1988
|
+
20000,
|
|
1989
|
+
json_path=out,
|
|
1990
|
+
)
|
|
1991
|
+
payload = json.loads(out.read_text())
|
|
1992
|
+
self.assertEqual(rc, 0) # advisory: never fails the run
|
|
1993
|
+
self.assertIn("completeness", payload["analyzers"])
|
|
1994
|
+
rec = payload["books"][0]
|
|
1995
|
+
self.assertIn("completeness", rec["verdicts"])
|
|
1996
|
+
self.assertEqual(rec["verdicts"]["completeness"]["status"], "OK")
|
|
1997
|
+
self.assertIn("prose 1/1", rec["verdicts"]["completeness"]["details"][0])
|
|
@@ -19,7 +19,7 @@ reserialize = [
|
|
|
19
19
|
|
|
20
20
|
[package.metadata]
|
|
21
21
|
requires-dist = [
|
|
22
|
-
{ name = "cquarry", specifier = ">=1.
|
|
22
|
+
{ name = "cquarry", specifier = ">=1.19.0" },
|
|
23
23
|
{ name = "html5lib", marker = "extra == 'reserialize'", specifier = ">=1.1" },
|
|
24
24
|
{ name = "tqdm" },
|
|
25
25
|
{ name = "vir-tui", specifier = ">=2.2.0" },
|
|
@@ -37,11 +37,11 @@ wheels = [
|
|
|
37
37
|
|
|
38
38
|
[[package]]
|
|
39
39
|
name = "cquarry"
|
|
40
|
-
version = "1.
|
|
40
|
+
version = "1.19.0"
|
|
41
41
|
source = { registry = "https://pypi.org/simple" }
|
|
42
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
42
|
+
sdist = { url = "https://files.pythonhosted.org/packages/d5/a1/7bb89bf5095c2dc64dc07029a540fb46af863fce1e7f042a479c25c46625/cquarry-1.19.0.tar.gz", hash = "sha256:34c87bd3c73957f9b32a54bd3fc6746820b40b003d333fffa2f678a288e403aa", size = 241248, upload-time = "2026-09-12T16:43:11.865Z" }
|
|
43
43
|
wheels = [
|
|
44
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
44
|
+
{ url = "https://files.pythonhosted.org/packages/2b/a7/36392bbfa1bebea7fad521ef01d93291df54b1343a2e9efd60eedd04556d/cquarry-1.19.0-py3-none-any.whl", hash = "sha256:7b0d417ac43218e1605db8c2458172f4ba688d8b12b3872a6e7ba03842fdcb98", size = 84386, upload-time = "2026-09-12T16:43:10.145Z" },
|
|
45
45
|
]
|
|
46
46
|
|
|
47
47
|
[[package]]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|