bindery-cli 0.27.0__tar.gz → 0.28.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/.gitignore +4 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/CLAUDE.md +3 -2
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/PKG-INFO +20 -10
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/README.md +13 -9
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/logo.svg +1 -1
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/patchnotes.md +100 -11
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/pyproject.toml +10 -1
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/roadmap.md +66 -7
- bindery_cli-0.28.0/scripts/FastSweep.java +102 -0
- bindery_cli-0.28.0/scripts/fast_sweep.py +160 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/spec.md +32 -9
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/__init__.py +2 -2
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/cli.py +42 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/epub.py +414 -2
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/transforms.py +52 -8
- bindery_cli-0.28.0/testing_facility/top500candidates/REPORT.md +20 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_cli.py +6 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_epub.py +290 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_transforms.py +46 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/uv.lock +14 -6
- bindery_cli-0.27.0/scripts/FastSweep.java +0 -40
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/.github/workflows/ci.yml +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/.github/workflows/publish.yml +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/AGENTS.md +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/LICENSE +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/docs/screenshots/library-sweep.png +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/run_tests.sh +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_block_in_inline.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_css_illegal_tags.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_duplicate_playorder.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_empty_body.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_illegal_tags.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_invalid_ids.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_invalid_st_tag.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_invalid_value_attr.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_missing_images.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/scripts/find_missing_title.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/__main__.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/audit.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/library.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/pagination.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/reserialize.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/validate.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/src/bindery/watermark.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/agda.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/agda_fixed.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/last_man_out.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/last_man_out_fixed.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/leaves_of_grass.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/leaves_of_grass_fixed.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/leaves_of_grass_st_fixed.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/leaves_of_grass_tags_fixed.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/muse_of_nightmares.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/muse_of_nightmares_fixed.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/poor_mans_fight.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/poor_mans_fight_fixed.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/test_block_in_inline.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/test_duplicate_playorder.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/test_empty_body.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/test_illegal_tags.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/test_invalid_ids.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/test_invalid_value.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/test_missing_title.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/unworthy_gods.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/test_facility/unworthy_gods_fixed.epub +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/testing_facility/books_to_fix/.gitkeep +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_audit.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_library.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_pagination.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_reserialize.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_validate.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_version.py +0 -0
- {bindery_cli-0.27.0 → bindery_cli-0.28.0}/tests/test_watermark.py +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# CLAUDE.md (
|
|
1
|
+
# CLAUDE.md (bindery-cli)
|
|
2
2
|
|
|
3
3
|
Per-project guidance. Overrides the global file where they conflict.
|
|
4
4
|
|
|
@@ -28,7 +28,8 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
|
|
|
28
28
|
visible content. Two kinds of exceptions exist, both opt-in and off unless requested:
|
|
29
29
|
* **Structural repairs** (`--fix-empty-body`, `--fix-missing-title`, `--fix-id-colons`,
|
|
30
30
|
`--fix-page-map`, `--strip-epub3-attrs`, `--downgrade-epub3-tags`,
|
|
31
|
-
`--unwrap-block-in-inline`, `--strip-invalid-value`, `--unwrap-illegal-tags
|
|
31
|
+
`--unwrap-block-in-inline`, `--strip-invalid-value`, `--unwrap-illegal-tags`,
|
|
32
|
+
`--prune-missing-resources`, `--strip-broken-anchors`, `--encode-url-spaces`;
|
|
32
33
|
transforms.py, threaded through epub.py): they alter markup structure or fabricate
|
|
33
34
|
minimal content. v0.14–v0.16 ran these unconditionally, which broke this rule;
|
|
34
35
|
v0.17.0 restored it. `--unwrap-illegal-tags` additionally protects any illegal-tag
|
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bindery-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.28.0
|
|
4
4
|
Summary: High-speed EPUB repair surgeon with a persistent Java validation daemon, safe HTML unwrapping, and native Calibre integration.
|
|
5
5
|
Author: vrnvctss
|
|
6
6
|
License: MIT
|
|
7
7
|
License-File: LICENSE
|
|
8
8
|
Keywords: calibre,ebook,epub,epubcheck,html-repair,repair,watermarks
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
14
|
+
Classifier: Topic :: Utilities
|
|
9
15
|
Requires-Python: >=3.14
|
|
10
16
|
Requires-Dist: cquarry>=1.9.0
|
|
11
17
|
Requires-Dist: tqdm
|
|
@@ -15,18 +21,18 @@ Requires-Dist: html5lib>=1.1; extra == 'reserialize'
|
|
|
15
21
|
Description-Content-Type: text/markdown
|
|
16
22
|
|
|
17
23
|
<div align="center">
|
|
18
|
-
<img src="logo.svg" width="96" height="96" alt="
|
|
19
|
-
<h1>
|
|
24
|
+
<img src="logo.svg" width="96" height="96" alt="bindery-cli logo"/>
|
|
25
|
+
<h1>bindery-cli</h1>
|
|
20
26
|
<p>Audit EPUB body text and repair broken EPUBs with safe, deterministic fixes, validated by epubcheck, with optional in-place replacement in a Calibre library.</p>
|
|
21
27
|
</div>
|
|
22
28
|
|
|
23
29
|
<p align="center">
|
|
24
|
-
<img src="docs/screenshots/library-sweep.png" alt="
|
|
30
|
+
<img src="docs/screenshots/library-sweep.png" alt="bindery-cli's dry-run library sweep: per-book epubcheck results and a summary table ending in 'no files written'">
|
|
25
31
|
</p>
|
|
26
32
|
|
|
27
33
|
## What it fixes
|
|
28
34
|
|
|
29
|
-
|
|
35
|
+
bindery-cli makes accidentally broken markup well-formed again. It does not rewrite or reflow content. The **default pass** applies only a small set of deterministic, semantics-preserving well-formedness fixes that real-world EPUBs (especially Calibre conversions) trip over — nothing else changes without an explicit flag:
|
|
30
36
|
|
|
31
37
|
- **Unclosed void elements** (`<link>`, `<br>`, `<img>`, ...) get self-closed.
|
|
32
38
|
- **Undeclared named entities** (` `, `°`, `é`, ...) become numeric character references that every XML parser understands.
|
|
@@ -54,6 +60,9 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
|
|
|
54
60
|
- **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package — `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive).
|
|
55
61
|
- **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents — `figure`/`section` to `div`, `figcaption` to `p` — keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed.
|
|
56
62
|
- **`--unwrap-illegal-tags`**: strips completely invalid or deprecated HTML tags that break EPUB3 validation (`<st>`, `<sentence>`, `<o>`, `<w>`, `<pagebreak>`) while retaining their inner text. Any of those names styled as an *element selector* by an EPUB stylesheet (`.css` entries and inline `<style>` blocks alike; class/id selectors like `.st`/`#w` don't count) is protected for the whole book, guaranteeing format preservation.
|
|
63
|
+
- **`--prune-missing-resources`**: removes references to files the archive does not contain (RSC-007/PKG-010): dead `<link>` elements, anchors' `href` to absent files (anchor text preserved), absent `<img>` sources (replaced by their alt text when they carry one), and orphaned non-spine OPF manifest items. Spine documents are never pruned: a missing spine document is reported as a damaged fragment, not silently dropped.
|
|
64
|
+
- **`--strip-broken-anchors`**: strips href attributes that cannot resolve, keeping the anchor text byte-for-byte: a `#fragment` the target document does not define (RSC-020/RSC-012; NCX navTargets fall back to the document target, so chapter navigation survives) and unresolvable URI schemes (`kindle:`, `file:`).
|
|
65
|
+
- **`--encode-url-spaces`**: percent-encodes raw spaces in `src`/`href` attribute values across the package (OPF manifest, NCX, content documents). A URL with a literal space is not a valid URL (RSC-020) and unresolvable on strict readers; the encoded form denotes the same file.
|
|
57
66
|
|
|
58
67
|
|
|
59
68
|
Three opt-in fixes are **lossy** and stand apart from the semantics-preserving rest:
|
|
@@ -77,7 +86,7 @@ The lossy modes (`--strip-pagination`, `--strip-broken-tags`, and `--strip-water
|
|
|
77
86
|
|
|
78
87
|
## Install
|
|
79
88
|
|
|
80
|
-
Python 3.14+, plus epubcheck on `PATH` for the gate. Dependencies: `tqdm` for progress output, plus the
|
|
89
|
+
Python 3.14+, plus epubcheck on `PATH` for the gate. Dependencies: `tqdm` for progress output, plus the VirInvictus libraries `vir-tui` (TUI rendering) and `cquarry` (read-only Calibre database access); `html5lib` remains an optional extra, needed only for `--reserialize`.
|
|
81
90
|
|
|
82
91
|
```sh
|
|
83
92
|
uv tool install bindery-cli # core tools
|
|
@@ -93,7 +102,7 @@ The sections below take each in turn.
|
|
|
93
102
|
|
|
94
103
|
## Auditing
|
|
95
104
|
|
|
96
|
-
|
|
105
|
+
bindery-cli includes a comprehensive auditing tool to inspect EPUB body text for non-schema flaws that epubcheck cannot catch. It extracts and analyzes the visible text to detect content issues, producing console reports that can be used to filter your library or feed into `bindery repair`.
|
|
97
106
|
|
|
98
107
|
Scan a Calibre library for specific issues:
|
|
99
108
|
|
|
@@ -178,7 +187,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
|
|
|
178
187
|
- `--only {fatals,ncx,all}` restricts the candidate set. `ncx` targets NCX-001 mismatches (detected without epubcheck); `fatals` needs `--audit`.
|
|
179
188
|
- `--id <ids>` limits the sweep to a comma-separated list of Calibre book IDs, skipping the full library walk.
|
|
180
189
|
- `--audit CSV` (the `fatals,errors,warnings,path` format produced by an epubcheck sweep) skips clean books so a run is fast. Paths are resolved on both sides, and a CSV that matches nothing triggers a loud warning instead of silently selecting zero books.
|
|
181
|
-
- `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice.
|
|
190
|
+
- `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice. bindery-cli launches a transparent, persistent Java daemon (`EpubcheckDaemon`) to evaluate candidates in the background, eliminating the JVM startup penalty. This reduces validation time from ~5 seconds per book down to ~0.05 seconds, allowing `--sweep` to scan a 5,000-book library in just 10 minutes. Combine with `--only fatals` for a self-contained, lightning-fast "find and fix the broken books" run.
|
|
182
191
|
- `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
|
|
183
192
|
- `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
|
|
184
193
|
- `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
|
|
@@ -191,9 +200,10 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
|
|
|
191
200
|
|
|
192
201
|
## Companion scripts
|
|
193
202
|
|
|
194
|
-
`scripts/` holds standalone, read-only utilities that are useful for EPUB maintenance but fall outside
|
|
203
|
+
`scripts/` holds standalone, read-only utilities that are useful for EPUB maintenance but fall outside bindery-cli's repair contract (fixing what they find would be a content change, which bindery-cli makes only via the opt-in `--strip-pagination`):
|
|
195
204
|
|
|
196
205
|
- `find_missing_images.py`: scans a library tree and reports every book whose `<img>` tags point at files that do not exist inside the archive (a common defect in converted EPUBs). Reads the archives in place; nothing is unpacked or written. The library path is set at the bottom of the script.
|
|
206
|
+
- `fast_sweep.py`: compiles and drives `FastSweep.java`, a parallel JVM epubcheck sweep that pays JVM startup once instead of per book (a 7,000-book dry-run sweep drops from hours to minutes). `--mode=audit` emits the `fatals,errors,warnings,path` CSV that `bindery library --audit` reads; `--mode=extract` emits per-book error codes for aggregation (`--summary` prints the per-code report). The epubcheck jar is located from the `epubcheck` launcher or `EPUBCHECK_JAR`; the harness compiles once (`--release 25`) and is cached on mtime.
|
|
197
207
|
|
|
198
208
|
## Development
|
|
199
209
|
|
|
@@ -209,7 +219,7 @@ MIT. See [LICENSE](LICENSE).
|
|
|
209
219
|
|
|
210
220
|
## Support
|
|
211
221
|
|
|
212
|
-
If
|
|
222
|
+
If bindery-cli's useful to you and you'd like to chip in:
|
|
213
223
|
|
|
214
224
|
- liberapay · [liberapay.com/bdkl](https://liberapay.com/bdkl/)
|
|
215
225
|
- bitcoin
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
<div align="center">
|
|
2
|
-
<img src="logo.svg" width="96" height="96" alt="
|
|
3
|
-
<h1>
|
|
2
|
+
<img src="logo.svg" width="96" height="96" alt="bindery-cli logo"/>
|
|
3
|
+
<h1>bindery-cli</h1>
|
|
4
4
|
<p>Audit EPUB body text and repair broken EPUBs with safe, deterministic fixes, validated by epubcheck, with optional in-place replacement in a Calibre library.</p>
|
|
5
5
|
</div>
|
|
6
6
|
|
|
7
7
|
<p align="center">
|
|
8
|
-
<img src="docs/screenshots/library-sweep.png" alt="
|
|
8
|
+
<img src="docs/screenshots/library-sweep.png" alt="bindery-cli's dry-run library sweep: per-book epubcheck results and a summary table ending in 'no files written'">
|
|
9
9
|
</p>
|
|
10
10
|
|
|
11
11
|
## What it fixes
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
bindery-cli makes accidentally broken markup well-formed again. It does not rewrite or reflow content. The **default pass** applies only a small set of deterministic, semantics-preserving well-formedness fixes that real-world EPUBs (especially Calibre conversions) trip over — nothing else changes without an explicit flag:
|
|
14
14
|
|
|
15
15
|
- **Unclosed void elements** (`<link>`, `<br>`, `<img>`, ...) get self-closed.
|
|
16
16
|
- **Undeclared named entities** (` `, `°`, `é`, ...) become numeric character references that every XML parser understands.
|
|
@@ -38,6 +38,9 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
|
|
|
38
38
|
- **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package — `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive).
|
|
39
39
|
- **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents — `figure`/`section` to `div`, `figcaption` to `p` — keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed.
|
|
40
40
|
- **`--unwrap-illegal-tags`**: strips completely invalid or deprecated HTML tags that break EPUB3 validation (`<st>`, `<sentence>`, `<o>`, `<w>`, `<pagebreak>`) while retaining their inner text. Any of those names styled as an *element selector* by an EPUB stylesheet (`.css` entries and inline `<style>` blocks alike; class/id selectors like `.st`/`#w` don't count) is protected for the whole book, guaranteeing format preservation.
|
|
41
|
+
- **`--prune-missing-resources`**: removes references to files the archive does not contain (RSC-007/PKG-010): dead `<link>` elements, anchors' `href` to absent files (anchor text preserved), absent `<img>` sources (replaced by their alt text when they carry one), and orphaned non-spine OPF manifest items. Spine documents are never pruned: a missing spine document is reported as a damaged fragment, not silently dropped.
|
|
42
|
+
- **`--strip-broken-anchors`**: strips href attributes that cannot resolve, keeping the anchor text byte-for-byte: a `#fragment` the target document does not define (RSC-020/RSC-012; NCX navTargets fall back to the document target, so chapter navigation survives) and unresolvable URI schemes (`kindle:`, `file:`).
|
|
43
|
+
- **`--encode-url-spaces`**: percent-encodes raw spaces in `src`/`href` attribute values across the package (OPF manifest, NCX, content documents). A URL with a literal space is not a valid URL (RSC-020) and unresolvable on strict readers; the encoded form denotes the same file.
|
|
41
44
|
|
|
42
45
|
|
|
43
46
|
Three opt-in fixes are **lossy** and stand apart from the semantics-preserving rest:
|
|
@@ -61,7 +64,7 @@ The lossy modes (`--strip-pagination`, `--strip-broken-tags`, and `--strip-water
|
|
|
61
64
|
|
|
62
65
|
## Install
|
|
63
66
|
|
|
64
|
-
Python 3.14+, plus epubcheck on `PATH` for the gate. Dependencies: `tqdm` for progress output, plus the
|
|
67
|
+
Python 3.14+, plus epubcheck on `PATH` for the gate. Dependencies: `tqdm` for progress output, plus the VirInvictus libraries `vir-tui` (TUI rendering) and `cquarry` (read-only Calibre database access); `html5lib` remains an optional extra, needed only for `--reserialize`.
|
|
65
68
|
|
|
66
69
|
```sh
|
|
67
70
|
uv tool install bindery-cli # core tools
|
|
@@ -77,7 +80,7 @@ The sections below take each in turn.
|
|
|
77
80
|
|
|
78
81
|
## Auditing
|
|
79
82
|
|
|
80
|
-
|
|
83
|
+
bindery-cli includes a comprehensive auditing tool to inspect EPUB body text for non-schema flaws that epubcheck cannot catch. It extracts and analyzes the visible text to detect content issues, producing console reports that can be used to filter your library or feed into `bindery repair`.
|
|
81
84
|
|
|
82
85
|
Scan a Calibre library for specific issues:
|
|
83
86
|
|
|
@@ -162,7 +165,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
|
|
|
162
165
|
- `--only {fatals,ncx,all}` restricts the candidate set. `ncx` targets NCX-001 mismatches (detected without epubcheck); `fatals` needs `--audit`.
|
|
163
166
|
- `--id <ids>` limits the sweep to a comma-separated list of Calibre book IDs, skipping the full library walk.
|
|
164
167
|
- `--audit CSV` (the `fatals,errors,warnings,path` format produced by an epubcheck sweep) skips clean books so a run is fast. Paths are resolved on both sides, and a CSV that matches nothing triggers a loud warning instead of silently selecting zero books.
|
|
165
|
-
- `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice.
|
|
168
|
+
- `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice. bindery-cli launches a transparent, persistent Java daemon (`EpubcheckDaemon`) to evaluate candidates in the background, eliminating the JVM startup penalty. This reduces validation time from ~5 seconds per book down to ~0.05 seconds, allowing `--sweep` to scan a 5,000-book library in just 10 minutes. Combine with `--only fatals` for a self-contained, lightning-fast "find and fix the broken books" run.
|
|
166
169
|
- `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
|
|
167
170
|
- `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
|
|
168
171
|
- `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
|
|
@@ -175,9 +178,10 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
|
|
|
175
178
|
|
|
176
179
|
## Companion scripts
|
|
177
180
|
|
|
178
|
-
`scripts/` holds standalone, read-only utilities that are useful for EPUB maintenance but fall outside
|
|
181
|
+
`scripts/` holds standalone, read-only utilities that are useful for EPUB maintenance but fall outside bindery-cli's repair contract (fixing what they find would be a content change, which bindery-cli makes only via the opt-in `--strip-pagination`):
|
|
179
182
|
|
|
180
183
|
- `find_missing_images.py`: scans a library tree and reports every book whose `<img>` tags point at files that do not exist inside the archive (a common defect in converted EPUBs). Reads the archives in place; nothing is unpacked or written. The library path is set at the bottom of the script.
|
|
184
|
+
- `fast_sweep.py`: compiles and drives `FastSweep.java`, a parallel JVM epubcheck sweep that pays JVM startup once instead of per book (a 7,000-book dry-run sweep drops from hours to minutes). `--mode=audit` emits the `fatals,errors,warnings,path` CSV that `bindery library --audit` reads; `--mode=extract` emits per-book error codes for aggregation (`--summary` prints the per-code report). The epubcheck jar is located from the `epubcheck` launcher or `EPUBCHECK_JAR`; the harness compiles once (`--release 25`) and is cached on mtime.
|
|
181
185
|
|
|
182
186
|
## Development
|
|
183
187
|
|
|
@@ -193,7 +197,7 @@ MIT. See [LICENSE](LICENSE).
|
|
|
193
197
|
|
|
194
198
|
## Support
|
|
195
199
|
|
|
196
|
-
If
|
|
200
|
+
If bindery-cli's useful to you and you'd like to chip in:
|
|
197
201
|
|
|
198
202
|
- liberapay · [liberapay.com/bdkl](https://liberapay.com/bdkl/)
|
|
199
203
|
- bitcoin
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
<svg xmlns="http://www.w3.org/2000/svg" width="128" height="128" viewBox="0 0 128 128" role="img" aria-label="
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="128" height="128" viewBox="0 0 128 128" role="img" aria-label="bindery-cli logo">
|
|
2
2
|
<rect width="128" height="128" rx="24" fill="#181616"/>
|
|
3
3
|
<!-- book spine / cover being rebound -->
|
|
4
4
|
<rect x="30" y="26" width="68" height="76" rx="5" fill="none" stroke="#c5c9c5" stroke-width="5"/>
|
|
@@ -1,4 +1,93 @@
|
|
|
1
|
-
#
|
|
1
|
+
# bindery-cli Patch Notes
|
|
2
|
+
## v0.28.0 (2026-09-05)
|
|
3
|
+
|
|
4
|
+
### Phase 12: repairs for the top epubcheck error codes
|
|
5
|
+
|
|
6
|
+
The 2026-09-05 full-library sweep (`testing_facility/top500candidates/REPORT.md`)
|
|
7
|
+
catalogued the top recurring codes: RSC-005 (164,302), RSC-020 (8,431), PKG-010
|
|
8
|
+
(4,112), RSC-007 (3,219), RSC-012 (1,940), HTM-025 (800). Bulk RSC-005 repair stays
|
|
9
|
+
rejected (the spec non-goal stands); everything else gained a deterministic,
|
|
10
|
+
epubcheck-gated, opt-in repair. The structural-repair set is now twelve; `--all`
|
|
11
|
+
includes all three.
|
|
12
|
+
|
|
13
|
+
- **`--prune-missing-resources`** (PKG-010/RSC-007) removes references to files the
|
|
14
|
+
archive does not contain: dead `<link>` elements (`dead_links_pruned`), anchors'
|
|
15
|
+
href to absent files with the text kept (`missing_file_hrefs_stripped`), absent
|
|
16
|
+
`<img>` sources replaced by their escaped alt text when they carry one
|
|
17
|
+
(`missing_imgs_unwrapped`) or dropped when they do not (`missing_imgs_pruned`), and
|
|
18
|
+
orphaned non-spine OPF manifest items (`manifest_items_pruned`). Spine documents are
|
|
19
|
+
never pruned: a missing spine document is a damaged fragment for the spine-integrity
|
|
20
|
+
report, not a pruning candidate.
|
|
21
|
+
- **`--strip-broken-anchors`** (RSC-020/RSC-012, plus the HTM-025 scheme half) strips
|
|
22
|
+
href attributes that cannot resolve while keeping the anchor text byte-for-byte: a
|
|
23
|
+
`#fragment` the target document does not define (`broken_fragment_hrefs_stripped`;
|
|
24
|
+
the anchor keeps its text, NCX navTargets fall back to the document target so
|
|
25
|
+
chapter navigation survives, and the fragment is never re-pointed at a guessed
|
|
26
|
+
sibling document), and non-resolvable URI schemes (`nonfile_scheme_hrefs_stripped`;
|
|
27
|
+
fixed resolvable set: http, https, mailto). NCX counter: `ncx_fragments_stripped`.
|
|
28
|
+
The id set for fragment checks is built from the documents as they will look when
|
|
29
|
+
the pass runs (core transforms, `--reserialize`, `--fix-id-colons` applied), so an
|
|
30
|
+
id rename can never make a valid reference look broken.
|
|
31
|
+
- **`--encode-url-spaces`** (the Death Masks shape: 67 RSC-020 "not a valid URL")
|
|
32
|
+
percent-encodes raw spaces in src/href attribute values across the package — OPF
|
|
33
|
+
manifest href, NCX content src, content documents (counter `url_spaces_encoded`).
|
|
34
|
+
Scope is fixed to the space character; archive entry names are untouched.
|
|
35
|
+
- **HTM-025 ampersand half closed by test:** `escape_bare_amp` always covered
|
|
36
|
+
attribute values (transforms run on raw text); the URL-attribute case is now pinned
|
|
37
|
+
by an explicit test instead of an assumption.
|
|
38
|
+
|
|
39
|
+
Fixture validation (all /tmp copies, gate-accepted, zero regressions):
|
|
40
|
+
|
|
41
|
+
| Book | Before | After | Fixes |
|
|
42
|
+
|---|---|---|---|
|
|
43
|
+
| Swann's Way | 0f/31e/1w | 0f/2e/0w | dead_links_pruned:29, nonfile_scheme_hrefs_stripped:1 |
|
|
44
|
+
| Sodom and Gomorrah | 0f/53e/0w | 0f/42e/0w | ncx_fragments_stripped:9, broken_fragment_hrefs_stripped:2 |
|
|
45
|
+
| Death Masks | 0f/68e/34w | 0f/1e/34w | url_spaces_encoded:67 |
|
|
46
|
+
|
|
47
|
+
Every surviving finding is RSC-005 (the non-goal) or the Death Masks entry-name
|
|
48
|
+
warnings (see below). The full label-vs-reality findings postscript lives in
|
|
49
|
+
roadmap.md Phase 12; headline: the staged "PKG-010" book has no missing resources at
|
|
50
|
+
all — its 34 PKG-010s are "file name contains spaces" warnings about the zip entry
|
|
51
|
+
names, which no shipped flag touches. Renaming entries and rewriting every reference
|
|
52
|
+
is a possible future flag.
|
|
53
|
+
|
|
54
|
+
### FastSweep: parallel JVM sweep harness (scripts/)
|
|
55
|
+
|
|
56
|
+
- `scripts/FastSweep.java` now merges the audit (CSV `fatals,errors,warnings,path`,
|
|
57
|
+
the format `bindery library --audit` reads) and extract (`path ||| CODE,...`)
|
|
58
|
+
prototypes behind `--mode=audit` / `--mode=extract`, keeping the stdin +
|
|
59
|
+
`parallelStream()` shape.
|
|
60
|
+
- **The prototype's extract mode never worked:** `CheckingReport` buffers every
|
|
61
|
+
message and writes nothing to its writer until `generate()` — and `generate()`
|
|
62
|
+
NPEs without a preceding `initialize()`. Every code list in the prototype's
|
|
63
|
+
`raw_errors*.txt` runs came back empty. The merged harness follows the CLI's own
|
|
64
|
+
lifecycle (`initialize` → validate → `generate`) and extracts codes from the JSON.
|
|
65
|
+
The committed REPORT.md's per-book code attributions should be treated as
|
|
66
|
+
unverified until regenerated with the fixed harness.
|
|
67
|
+
- `scripts/fast_sweep.py` (stdlib-only) is the wrapper: locates the epubcheck jar
|
|
68
|
+
(`--jar` / `EPUBCHECK_JAR` / parsed from the launcher), compiles the harness once
|
|
69
|
+
with `--release 25` (cached on mtime), scans a directory or a path list, and
|
|
70
|
+
`--summary` aggregates extract output into the per-code REPORT-style table.
|
|
71
|
+
Measured on the 14 staged fixtures: ~4.5 s across all cores vs ~42 s sequential.
|
|
72
|
+
- The audit CSV feeds `bindery library --audit` cleanly (smoke-tested for candidate
|
|
73
|
+
selection both ways).
|
|
74
|
+
|
|
75
|
+
### Notes
|
|
76
|
+
|
|
77
|
+
- epubcheck 5.3.0 surfaces disagree on counts: the human listing prints raw message
|
|
78
|
+
totals while the `--json` checker block prints deduplicated ones (Swann's Way: 31
|
|
79
|
+
vs 2 errors). bindery's gate reads the JSON surface consistently for both
|
|
80
|
+
measurements, so gate decisions are unaffected. FastSweep's audit mode carries the
|
|
81
|
+
raw CheckingReport counts; that CSV is used for candidate selection only.
|
|
82
|
+
- Related, unchanged on purpose: validate.py's persistent daemon launches
|
|
83
|
+
`java -cp ".:epubcheck.jar"` without epubcheck's `lib/` dependencies, so it always
|
|
84
|
+
fails its first check and silently falls back to the subprocess JSON path on this
|
|
85
|
+
machine. Fixing the classpath would flip gate measurements from deduplicated to
|
|
86
|
+
raw counts — a gate-behavior change that gets its own decision.
|
|
87
|
+
- Tests 260 → 277: transform units (encoding, protection, idempotency, HTM-025),
|
|
88
|
+
repair-level opt-in/counter/output tests for all three flags, the structural
|
|
89
|
+
default-off case list, and the CLI wiring tuples.
|
|
90
|
+
|
|
2
91
|
## v0.27.0 (2026-09-02)
|
|
3
92
|
|
|
4
93
|
### RSC-005 verdict, part two: --downgrade-epub3-tags
|
|
@@ -172,7 +261,7 @@
|
|
|
172
261
|
directories and case-insensitive matching behave exactly as before. The
|
|
173
262
|
`(123)` directory-name regex remains the documented no-catalog fallback.
|
|
174
263
|
The `audit.py` raw-join swap was waived at the Phase 9 design session (its
|
|
175
|
-
three targeted joins are cheaper than whole-library hydration and
|
|
264
|
+
three targeted joins are cheaper than whole-library hydration and bindery-cli
|
|
176
265
|
wants per-book maps, not row dicts); do not "improve" it into a regression.
|
|
177
266
|
- **Dependency**: requires cquarry ≥ 1.8 (`format_path_index`). The `uv.lock`
|
|
178
267
|
still pins the pre-1.8 `@main` commit until cquarry's 1.8.0 commits are
|
|
@@ -206,7 +295,7 @@
|
|
|
206
295
|
- **Dependency bump (deliberate pin move).** The `vir-tui` pin moved from `d13ad0e` to `e53f17e`
|
|
207
296
|
(vir-tui 2.2.0), per the repo policy of pinning exact commits and bumping deliberately. 2.2.0 is
|
|
208
297
|
additive — `progress_box()`, `interactive_session()`, `prompt_float`/`prompt_path`/`confirm`,
|
|
209
|
-
`out_note`, `text_mode()`, and results-pager search — and every API
|
|
298
|
+
`out_note`, `text_mode()`, and results-pager search — and every API bindery-cli uses (`ui.tqdm`,
|
|
210
299
|
`ui.info`, `ui.print_header`) is unchanged, so no behavior change is expected; `uv.lock`
|
|
211
300
|
regenerated to match the new pin.
|
|
212
301
|
- **Version sync fix.** `src/bindery/__init__.py` still said `0.18.0` while `pyproject.toml` said
|
|
@@ -253,7 +342,7 @@
|
|
|
253
342
|
- **Tagged books are queued for OPF regeneration.** The pinned cquarry bump (1.1 → 1.2) means every
|
|
254
343
|
`WritableCalibreDB.add_tag()` now records the book id in Calibre's `metadata_dirtied` queue —
|
|
255
344
|
the table upstream consumes to decide which sidecar `.opf`s to regenerate (and re-push to wireless
|
|
256
|
-
readers) at next startup. Previously a
|
|
345
|
+
readers) at next startup. Previously a bindery-cli-applied tag would appear in the GUI but never reach
|
|
257
346
|
OPF/wireless sync. Verified end-to-end against a synthetic library: flagged book tagged,
|
|
258
347
|
`metadata_dirtied` gains exactly that id, clean books untouched.
|
|
259
348
|
- **Requires `cquarry` @ e19c24c** (v1.2.0); pin bumped accordingly.
|
|
@@ -326,13 +415,13 @@ above the daemon call where it is an actual docstring. Suite: 183 -> 201 tests.
|
|
|
326
415
|
|
|
327
416
|
### Core Upgrades
|
|
328
417
|
|
|
329
|
-
**Centralized Calibre DB Access:**
|
|
418
|
+
**Centralized Calibre DB Access:** bindery-cli has transitioned to the unified `cquarry` library for all read-only `metadata.db` accesses. The internal `bindery.audit` DB connection logic has been completely replaced with `cquarry.db.CalibreDB`. This inherits robust Calibre lock handling, database snapshotting, and ensures query logic stays perfectly synchronized with `CalibreQuarry` and `Hermitage`.
|
|
330
419
|
|
|
331
420
|
|
|
332
421
|
## v0.15.0 (2026-08-23)
|
|
333
422
|
|
|
334
423
|
### Features
|
|
335
|
-
- **Integrated EPUB Auditing (`bindery audit`):** Successfully absorbed the external `audit_epub` codebase directly into
|
|
424
|
+
- **Integrated EPUB Auditing (`bindery audit`):** Successfully absorbed the external `audit_epub` codebase directly into bindery-cli as a native subcommand. This transforms bindery-cli from a pure repair tool into a complete EPUB lifecycle toolkit (Audit -> Repair -> Validate).
|
|
336
425
|
- **`content`:** Identifies non-English text by sniffing the text blocks, reporting the predominant script block (e.g., Cyrillic, CJK) to catch untranslated or mis-encoded EPUBs.
|
|
337
426
|
- **`pagenumbers`:** Detects books polluted with hardcoded print page numbers (using sliding window regex to find high-density sequential digits interrupting prose), seamlessly bridging into `bindery repair --strip-pagination` for the fix.
|
|
338
427
|
- **`emptytext`:** Audits EPUBs to find effectively empty books or spine stubs (below configurable `--min-chars` and `--thin-chars` thresholds).
|
|
@@ -355,7 +444,7 @@ above the daemon call where it is an actual docstring. Suite: 183 -> 201 tests.
|
|
|
355
444
|
## v0.13.0 (2026-08-22)
|
|
356
445
|
|
|
357
446
|
### Structural Changes
|
|
358
|
-
- **Epubcheck Daemon:** Radically accelerated the `bindery library` validation gate by implementing a transparent, persistent Java daemon.
|
|
447
|
+
- **Epubcheck Daemon:** Radically accelerated the `bindery library` validation gate by implementing a transparent, persistent Java daemon. bindery-cli now automatically compiles `FastDaemon.java` in the background and pipes EPUB paths to it, eliminating the JVM startup penalty.
|
|
359
448
|
- Validation time dropped from ~5 seconds per book down to under ~0.05 seconds per book, cutting library sweeps from 6 hours to 10 minutes.
|
|
360
449
|
|
|
361
450
|
## v0.12.3 (2026-08-22)
|
|
@@ -378,10 +467,10 @@ above the daemon call where it is an actual docstring. Suite: 183 -> 201 tests.
|
|
|
378
467
|
## v0.12.0 (2026-08-22)
|
|
379
468
|
|
|
380
469
|
### Features
|
|
381
|
-
- **Oceanstrip Merged:** Fully incorporated the `oceanstrip` standalone tool directly into
|
|
470
|
+
- **Oceanstrip Merged:** Fully incorporated the `oceanstrip` standalone tool directly into bindery-cli's primary pipeline.
|
|
382
471
|
- Added `--strip-watermarks` flag to systematically detect and remove producer/distributor watermarks (e.g. OceanofPDF stamps and zero-byte marker files, ABC Amber LIT Converter injections) from EPUBs.
|
|
383
472
|
- This lossy transform is correctly governed by the `validate.no_worse` EPUBCheck oracle, ensuring clean structural extraction without regressing file validity.
|
|
384
|
-
- The `--strip-watermarks` argument is automatically engaged when running
|
|
473
|
+
- The `--strip-watermarks` argument is automatically engaged when running bindery-cli with `--all`.
|
|
385
474
|
|
|
386
475
|
## v0.11.0 (2026-08-21)
|
|
387
476
|
|
|
@@ -493,7 +582,7 @@ that the correct hook does not create.
|
|
|
493
582
|
|
|
494
583
|
Two constraints recorded with the decision, because they shape what the plugin can
|
|
495
584
|
be: it runs inside Calibre's bundled Python, so it must vendor its own code and
|
|
496
|
-
cannot assume
|
|
585
|
+
cannot assume bindery-cli is installed (the minimal-dependency rule is what makes this
|
|
497
586
|
practical, and the optional `html5lib` path cannot come along); and epubcheck
|
|
498
587
|
cannot gate an import, being an external Java process of seconds per book, so the
|
|
499
588
|
plugin does the deterministic transforms and leaves gated work to the CLI where a
|
|
@@ -643,7 +732,7 @@ round of CLI hardening and UX. Every fix ships with a unittest regression test.
|
|
|
643
732
|
|
|
644
733
|
## v0.5.0 (2026-06-14)
|
|
645
734
|
|
|
646
|
-
**New: `--strip-pagination` (opt-in, lossy).** Removes print page numbers and running headers that a PDF/OCR conversion baked into the body text as literal paragraphs, which reflow into the middle of sentences ("where the hay cart 16 was taking him"). This is the first mode that removes visible content, so it is a deliberate, fenced-off exception to
|
|
735
|
+
**New: `--strip-pagination` (opt-in, lossy).** Removes print page numbers and running headers that a PDF/OCR conversion baked into the body text as literal paragraphs, which reflow into the middle of sentences ("where the hay cart 16 was taking him"). This is the first mode that removes visible content, so it is a deliberate, fenced-off exception to bindery-cli's semantics-preserving rule: off by default, and accepted by a new `no_worse` bar (no net-new fatals or errors) instead of the improvement-demanding `gate`, because a baked page number is valid markup that epubcheck cannot see.
|
|
647
736
|
|
|
648
737
|
- Removes only injected furniture, never the author's prose. Where a number split a sentence it rejoins the two paragraphs (closing up a word split like `compli-`/`mentary`); page-list `id` anchors are hoisted into the merged paragraph so navigation still resolves.
|
|
649
738
|
- A book is treated as paginated only when it has both a dense run of standalone arabic numbers (>= 20) and several confident mid-sentence interrupts (>= 3), so a merely chapter-numbered book is never touched. Roman chapter/front-matter numerals and year-range values are preserved.
|
|
@@ -4,9 +4,17 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "bindery-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.28.0"
|
|
8
8
|
description = "High-speed EPUB repair surgeon with a persistent Java validation daemon, safe HTML unwrapping, and native Calibre integration."
|
|
9
9
|
readme = "README.md"
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 5 - Production/Stable",
|
|
12
|
+
"Environment :: Console",
|
|
13
|
+
"Intended Audience :: End Users/Desktop",
|
|
14
|
+
"License :: OSI Approved :: MIT License",
|
|
15
|
+
"Programming Language :: Python :: 3.14",
|
|
16
|
+
"Topic :: Utilities",
|
|
17
|
+
]
|
|
10
18
|
requires-python = ">=3.14"
|
|
11
19
|
keywords = ["epub", "epubcheck", "calibre", "repair", "ebook", "html-repair", "watermarks"]
|
|
12
20
|
license = { text = "MIT" }
|
|
@@ -45,3 +53,4 @@ packages = ["src/bindery"]
|
|
|
45
53
|
# negated expression.
|
|
46
54
|
select = ["E4", "E7", "E9", "F", "I", "UP", "B"]
|
|
47
55
|
|
|
56
|
+
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# bindery-cli roadmap
|
|
2
2
|
|
|
3
3
|
## Phase 1: deterministic repair engine (shipped, v0.1.0)
|
|
4
4
|
|
|
@@ -303,14 +303,14 @@ enters the library, the user's original on disk is never touched, and no write t
|
|
|
303
303
|
|
|
304
304
|
Three consequences worth stating, because they change existing plans:
|
|
305
305
|
|
|
306
|
-
1. **
|
|
306
|
+
1. **bindery-cli's separate "metadata.db nudge" item is unnecessary and is dropped.**
|
|
307
307
|
It existed so Calibre would notice a changed file size after an in-place
|
|
308
308
|
replacement. Under `on_import` the file is modified *before* Calibre reads it,
|
|
309
309
|
so the size it records is already correct. The nudge was solving a problem
|
|
310
310
|
created by replacing files behind Calibre's back.
|
|
311
311
|
2. **The plugin must carry its own code.** It runs inside Calibre's bundled
|
|
312
312
|
Python, which will not have either tool pip-installed. Both being minimal-dependency
|
|
313
|
-
is what makes this practical: the module vendors into the plugin zip.
|
|
313
|
+
is what makes this practical: the module vendors into the plugin zip. bindery-cli's
|
|
314
314
|
optional `html5lib` path and its epubcheck gate cannot come along, so the
|
|
315
315
|
plugin must degrade honestly rather than half-run.
|
|
316
316
|
3. **epubcheck cannot gate the import.** It is an external Java process taking
|
|
@@ -339,10 +339,10 @@ never wrote — exactly the class of fix that must stay opt-in under the safety
|
|
|
339
339
|
|
|
340
340
|
## Phase 4: cquarry Integration (complete 2026-08-28, v0.19.0)
|
|
341
341
|
|
|
342
|
-
With the transition to the `cquarry` shared library (v0.16.0),
|
|
342
|
+
With the transition to the `cquarry` shared library (v0.16.0), bindery-cli inherits the ability to perform complex search and batch resolution natively. As the `cquarry` library adds new write capabilities, bindery-cli is slated to adopt the following upgrades:
|
|
343
343
|
|
|
344
344
|
- [x] **Format Path Resolution:** Transition to `cquarry.get_format_path()` to remove manual path concatenations during file discovery. *(done: v0.18.0 for library-mode audits; v0.19.0 finishes the job — `CalibreIdResolver` builds the id→EPUB-path map through `get_format_path()` so `--install-to-calibre` resolves the book id from `metadata.db` instead of the `(id)` directory fragment.)*
|
|
345
|
-
- [x] **Safe Tag Application:** When `cquarry` implements safe DB writes (`add_tag`),
|
|
345
|
+
- [x] **Safe Tag Application:** When `cquarry` implements safe DB writes (`add_tag`), bindery-cli will gain the ability to automatically tag books in the Calibre UI as "Audited" or "Flagged" when issues are found, rather than relying strictly on console output. *(done: v0.18.1 — `audit --tag` applies via `cquarry.write.WritableCalibreDB.add_tag`, OPF-resync queued through `metadata_dirtied`; v0.19.0 extends `--tag` to the new `--id` single-book mode.)*
|
|
346
346
|
- [x] **Single-Entity Fetching:** Utilize `get_book(book_id)` for faster isolated audits when analyzing a single book, avoiding the overhead of caching the entire library layout. *(done: v0.19.0 — `audit --id BOOK_ID` fetches one row via `get_book()`, resolves the EPUB via `get_format_path()`, and reports the same verdicts as directory mode.)*
|
|
347
347
|
|
|
348
348
|
## Phase 6: Code Sweep & Fixes (2026-08-23)
|
|
@@ -527,7 +527,7 @@ aside), the sweep's sub-reason split, the flipped CRC test plus
|
|
|
527
527
|
hand-rolled zipfile sweep.
|
|
528
528
|
|
|
529
529
|
### Pending Repair Implementations
|
|
530
|
-
- **Strip Invalid/Deprecated Attributes (RSC-005):** `testing_facility/books_to_fix/Lost Lambs (Madeline Cash) (z-library.sk, 1lib.sk, z-lib.sk).epub` throws `RSC-005` errors for `page-progression-direction` on `<package>`, `epub:type` on `<body>`, `aria-label` outside of allowed namespaces, and unwrapping illegal `<span>` tags.
|
|
530
|
+
- **Strip Invalid/Deprecated Attributes (RSC-005):** `testing_facility/books_to_fix/Lost Lambs (Madeline Cash) (z-library.sk, 1lib.sk, z-lib.sk).epub` throws `RSC-005` errors for `page-progression-direction` on `<package>`, `epub:type` on `<body>`, `aria-label` outside of allowed namespaces, and unwrapping illegal `<span>` tags. bindery-cli currently intercepts `<li value="..">` via `--strip-invalid-value`, but needs a broader pass to scrub these specific deprecated HTML5/EPUB3 attributes when targeting EPUB2 compliance, or just to silence strict validation errors. **DECISION 2026-09-02 (Brandon, on the NEW-AUDIT brief): Option B — the attribute scrub ships as its own opt-in flag (v0.26.0), and the element downgrade ships separately with the CSS-protection work (v0.27.0); no per-book flags, no silent tolerance.**
|
|
531
531
|
*(Attribute scrub shipped in v0.26.0 as `--strip-epub3-attrs` (counter
|
|
532
532
|
`epub3_attrs_stripped`): the three named attributes, fixed documented set,
|
|
533
533
|
content docs and the OPF both, lookalikes survive. The illegal `<span>`
|
|
@@ -555,7 +555,7 @@ subprocess is gone, so the crash class no longer exists.)*
|
|
|
555
555
|
|
|
556
556
|
## Phase 11: Migrate install-to-calibre to native cquarry API (proposed 2026-08-31)
|
|
557
557
|
|
|
558
|
-
*Context: `bindery library --install-to-calibre` currently shells out to the external `calibredb add_format` CLI binary. This is fragile (it crashed on 2026-08-31 due to a non-existent `--replace` flag) and bypasses the transaction and trigger-safety guarantees built into the `cquarry` library. Since
|
|
558
|
+
*Context: `bindery library --install-to-calibre` currently shells out to the external `calibredb add_format` CLI binary. This is fragile (it crashed on 2026-08-31 due to a non-existent `--replace` flag) and bypasses the transaction and trigger-safety guarantees built into the `cquarry` library. Since bindery-cli already imports `cquarry` for path resolution, it should adopt the native write module.*
|
|
559
559
|
|
|
560
560
|
- [x] **Adopt `WritableCalibreDB` for format installation:** Replace the `subprocess.run(["calibredb", ...])` call in `bindery/library.py` with `cquarry.write.WritableCalibreDB.add_format()`. This keeps all database writes safely centralized in `cquarry` and eliminates the dependency on the external Calibre CLI for structural sweeps.
|
|
561
561
|
*(Shipped in v0.24.0 as `install_format()`: the file is placed atomically
|
|
@@ -570,3 +570,62 @@ subprocess is gone, so the crash class no longer exists.)*
|
|
|
570
570
|
degrades the same way rather than losing the repair. `uv.lock` moved to
|
|
571
571
|
cquarry 1.9.0 in the same release, closing NEW-AUDIT Stage 1's pending
|
|
572
572
|
note. Tests 240 → 244.)*
|
|
573
|
+
|
|
574
|
+
## Phase 12: Top epubcheck error resolution from full-library sweep (proposed 2026-09-05)
|
|
575
|
+
|
|
576
|
+
*Context: A diagnostic sweep of the full Calibre library on 2026-09-05 (`testing_facility/top500candidates/REPORT.md`) catalogued the top recurring error codes across all candidates: RSC-005 (164,302), RSC-020 (8,431), PKG-010 (4,112), RSC-007 (3,219), RSC-012 (1,940), and HTM-025 (800). Evaluating these against spec.md establishes which are safe to automate deterministically and which must remain rejected under the safety charter.*
|
|
577
|
+
|
|
578
|
+
- [x] **Opt-in missing resource pruning (`--prune-missing-resources`):**
|
|
579
|
+
- **PKG-010 (missing package resource):** When the OPF manifest lists an `<item>` whose file is absent from the archive, prune the orphaned declaration. Strictly scoped to non-spine assets (fonts, stylesheets, unused media); missing spine documents remain classified as damaged fragments and are never silently dropped.
|
|
580
|
+
- **RSC-007 (referenced resource not found):** When content documents reference absent external assets, prune dead `<link rel="stylesheet">` tags and strip `href` attributes from anchors targeting non-existent files. Missing `<img>` elements are pruned or replaced with their `alt` text under character conservation.
|
|
581
|
+
*(Shipped in v0.28.0. Fixture reality check: Swann's Way, the staged RSC-007 book, carries exactly one defect shape — 29 identical `<link rel="stylesheet" href="../page-template.xpgt">` elements for an Adobe page template the converter never copied in; all 29 prune, zero regression. Death Masks, the staged PKG-010 book, has NO missing resources at all: its 34 PKG-010s are "file name contains spaces" warnings about the zip entry names, not absent files, so nothing is prunable there — see the `--encode-url-spaces` note below. Counters: `manifest_items_pruned`, `dead_links_pruned`, `missing_file_hrefs_stripped`, `missing_imgs_unwrapped`, `missing_imgs_pruned`.)*
|
|
582
|
+
- [x] **Opt-in broken anchor stripping (`--strip-broken-anchors`):**
|
|
583
|
+
- **RSC-020 (fragment identifier not defined):** When an `href` points to a non-existent `#fragment` target, remove the `href` attribute rather than fabricating an ID. Inner anchor text is preserved byte-for-byte under character conservation.
|
|
584
|
+
- **RSC-012 (fragment points to wrong element):** For content anchors targeting illegal elements, strip the invalid fragment. For NCX `<content src="doc.xhtml#bad_id"/>` references, strip the fragment to point directly to the containing document, preserving chapter navigation.
|
|
585
|
+
*(Shipped in v0.28.0. Sodom and Gomorrah, the staged RSC-012 book, is Mobipocket `filepos` drift: the converter re-split the flow and every NCX target points one file late. Per the charter the repair strips the fragment and keeps the document target — it never re-points at the guessed sibling document, because the 9th navPoint's id exists nowhere at all. Fixture run: 9 NCX fragments + 2 in-content Notes anchors stripped, 11 RSC-012 cleared, zero regression. Counters: `broken_fragment_hrefs_stripped`, `nonfile_scheme_hrefs_stripped`, `ncx_fragments_stripped`.)*
|
|
586
|
+
- [x] **URI and entity normalization (HTM-025):**
|
|
587
|
+
- **Attribute ampersand escaping:** Ensure `transforms.escape_bare_amp` covers bare ampersands inside URL attributes (such as query strings `?a=1&b=2` converted to `?a=1&b=2`).
|
|
588
|
+
- **Non-registered URI schemes:** Strip unresolvable proprietary or local URI schemes (e.g. `scrivcmt://`, local `file:///` paths) from anchor `href` attributes under `--strip-broken-anchors`, retaining visible prose.
|
|
589
|
+
*(Shipped in v0.28.0. The ampersand half already worked — the transforms run on raw text, so attribute values were always covered; the behavior is now pinned by an explicit test rather than an assumption. The scheme half ships inside `--strip-broken-anchors` with a fixed resolvable set (http, https, mailto); Swann's Way's single HTM-025 was a `kindle:embed:` anchor href, stripped with the anchor text kept.)*
|
|
590
|
+
- [x] **Fast sweep testing harness (`FastSweep.java` JVM parallelization):**
|
|
591
|
+
- **Objective:** Integrate parallel JVM validation sweeps directly into the testing/benchmarking suite to bypass Python subprocess I/O bottlenecks during massive 7,000+ book dry-runs.
|
|
592
|
+
- **Context:** Standard `epubcheck` execution in a Python `subprocess.run` loop takes ~3 seconds per book (6.5 hours for the library). The `FastSweepExtract.java` prototype proved that instantiating `com.adobe.epubcheck.api.EpubCheck` directly and routing `CheckingReport` to a `StringWriter` within a Java `parallelStream()` reduces this to minutes by sharing the JVM startup cost and saturating all CPU cores.
|
|
593
|
+
- **Implementation Details for Future Agents:**
|
|
594
|
+
1. Expand `scripts/FastSweep.java` (or merge `FastSweepExtract.java` into it) to parse CLI flags (e.g., `--mode=audit`, `--mode=extract`).
|
|
595
|
+
2. Write an integration wrapper in `bindery-cli` that compiles the Java file (targeting `--release 25`) and invokes it against a provided directory.
|
|
596
|
+
3. **Test Fixtures:** Use the sample candidate EPUBs pre-staged in `testing_facility/top500candidates/` (which includes confirmed instances of `RSC-005`, `RSC-020`, `PKG-010`, `RSC-007`, `RSC-012`, and `HTM-025`) to verify the Java AST parsing and error-code extraction before running it across the live library.
|
|
597
|
+
*(Shipped in v0.28.0 as scripts/FastSweep.java + scripts/fast_sweep.py — see patchnotes for the full notes; the prototype's extract mode never actually worked: CheckingReport buffers its messages and writes nothing until generate(), which NPEs without a preceding initialize(); the merged harness follows the CLI's own initialize → validate → generate lifecycle and extracts codes from the JSON. The two prototypes' stdin + parallelStream shape was otherwise kept. Measured on the 14 staged fixtures: 14 books in ~4.5 s (all cores) vs ~3 s/book sequential.)*
|
|
598
|
+
|
|
599
|
+
### Phase 12 findings postscript (2026-09-05)
|
|
600
|
+
|
|
601
|
+
The REPORT.md code labels do not describe the actual defects in the staged fixtures;
|
|
602
|
+
the underlying conditions are what the repairs target:
|
|
603
|
+
|
|
604
|
+
- **Death Masks ("RSC-020 fragment not defined" + "PKG-010 resource missing"):** all 39
|
|
605
|
+
manifest items exist. The 67 RSC-020s are "not a valid URL" (literal spaces in OPF
|
|
606
|
+
hrefs and NCX srcs; fixed by `--encode-url-spaces`), and the 34 PKG-010s are
|
|
607
|
+
"file name contains spaces" WARNINGS about the zip entry names themselves. Clearing
|
|
608
|
+
those would mean renaming archive entries and rewriting every reference — a viable
|
|
609
|
+
future flag, deliberately not attempted here. After repair: 0f/1e/34w (the one error
|
|
610
|
+
is an `<img id="Picture 0">` RSC-005, out of scope).
|
|
611
|
+
- **Swann's Way ("RSC-007 missing resource" + HTM-025):** 29 identical dead Adobe
|
|
612
|
+
page-template `<link>` elements plus one `kindle:embed:` anchor. After
|
|
613
|
+
`--prune-missing-resources --strip-broken-anchors`: 0f/2e/0w.
|
|
614
|
+
- **Sodom and Gomorrah (RSC-012):** Mobipocket `filepos` anchors pointing one split
|
|
615
|
+
file late. After `--strip-broken-anchors`: 0f/42e/0w (the 42 survivors are RSC-005,
|
|
616
|
+
the declared non-goal).
|
|
617
|
+
- **epubcheck 5.3.0 surfaces disagree on counts:** the human listing reports raw
|
|
618
|
+
message totals (Swann's Way: 31 errors) while the `--json` checker block reports
|
|
619
|
+
deduplicated ones (2). bindery's gate reads the JSON surface consistently for both
|
|
620
|
+
measurements, so gate decisions are unaffected — but audit CSVs produced by
|
|
621
|
+
FastSweep's audit mode carry the raw CheckingReport counts, which are larger than
|
|
622
|
+
the JSON surface the gate re-measures with. Candidate selection only (fatals > 0,
|
|
623
|
+
clean-book skipping), so the mixing is safe today.
|
|
624
|
+
- Related discovery, unchanged on purpose: `validate.py`'s persistent daemon launches
|
|
625
|
+
`java -cp ".:epubcheck.jar"` without epubcheck's `lib/` dependencies, so on this
|
|
626
|
+
machine it has always failed its first check and silently fallen back to the
|
|
627
|
+
subprocess JSON path. Fixing the classpath would switch gate measurements from the
|
|
628
|
+
deduplicated JSON counts to raw CheckingReport counts — a behavior change to the
|
|
629
|
+
gate that deserves its own decision, not a drive-by edit.
|
|
630
|
+
|
|
631
|
+
Non-goals: No bulk repair of RSC-005 schema/content-model violations. With 164,302 instances, arbitrary schema repair violates the explicit non-goal in spec.md ("Fixing RSC-005 schema/content-model violations in bulk"). Restructuring arbitrary markup requires semantic human judgment; attempting to automate it risks silent content corruption, tag swallowing, or destroying valid styling. Specific, narrowly bounded RSC-005 sub-cases remain opt-in under their own flags (`--strip-epub3-attrs`, `--downgrade-epub3-tags`, `--unwrap-block-in-inline`, `--strip-invalid-value`, `--unwrap-illegal-tags`, `--add-img-alt`, `--fix-id-colons`); arbitrary bulk repair is rejected. No spine-item manifest pruning (missing spine documents indicate corrupt archives or broken fragments, not safe pruning candidates). No text deletion in anchor unwrapping (character conservation is absolute).
|