bindery-cli 0.31.0__tar.gz → 0.33.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/.gitignore +5 -0
  2. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/CLAUDE.md +20 -5
  3. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/PKG-INFO +24 -12
  4. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/README.md +23 -11
  5. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/patchnotes.md +206 -0
  6. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/pyproject.toml +1 -1
  7. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/roadmap.md +321 -30
  8. bindery_cli-0.33.0/run_tests.sh +18 -0
  9. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/spec.md +53 -7
  10. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/__init__.py +1 -1
  11. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/audit.py +245 -208
  12. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/cli.py +94 -10
  13. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/epub.py +138 -35
  14. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/library.py +78 -6
  15. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/pagination.py +16 -4
  16. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/reserialize.py +18 -4
  17. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/transforms.py +122 -32
  18. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/validate.py +177 -19
  19. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/watermark.py +67 -17
  20. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/tests/test_audit.py +506 -67
  21. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/tests/test_cli.py +182 -10
  22. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/tests/test_epub.py +207 -1
  23. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/tests/test_library.py +120 -3
  24. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/tests/test_pagination.py +52 -0
  25. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/tests/test_reserialize.py +13 -0
  26. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/tests/test_transforms.py +126 -0
  27. bindery_cli-0.33.0/tests/test_validate.py +305 -0
  28. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/tests/test_watermark.py +100 -17
  29. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/uv.lock +1 -1
  30. bindery_cli-0.31.0/run_tests.sh +0 -10
  31. bindery_cli-0.31.0/scripts/FastSweepExtract.java +0 -44
  32. bindery_cli-0.31.0/scripts/find_block_in_inline.py +0 -44
  33. bindery_cli-0.31.0/scripts/find_css_illegal_tags.py +0 -64
  34. bindery_cli-0.31.0/scripts/find_duplicate_playorder.py +0 -50
  35. bindery_cli-0.31.0/scripts/find_empty_body.py +0 -44
  36. bindery_cli-0.31.0/scripts/find_illegal_tags.py +0 -169
  37. bindery_cli-0.31.0/scripts/find_invalid_ids.py +0 -53
  38. bindery_cli-0.31.0/scripts/find_invalid_st_tag.py +0 -43
  39. bindery_cli-0.31.0/scripts/find_invalid_value_attr.py +0 -46
  40. bindery_cli-0.31.0/scripts/find_missing_images.py +0 -81
  41. bindery_cli-0.31.0/scripts/find_missing_title.py +0 -46
  42. bindery_cli-0.31.0/scripts/sweep.sh +0 -34
  43. bindery_cli-0.31.0/test_facility/agda.epub +0 -0
  44. bindery_cli-0.31.0/test_facility/agda_fixed.epub +0 -0
  45. bindery_cli-0.31.0/test_facility/last_man_out.epub +0 -0
  46. bindery_cli-0.31.0/test_facility/last_man_out_fixed.epub +0 -0
  47. bindery_cli-0.31.0/test_facility/leaves_of_grass.epub +0 -0
  48. bindery_cli-0.31.0/test_facility/leaves_of_grass_fixed.epub +0 -0
  49. bindery_cli-0.31.0/test_facility/leaves_of_grass_st_fixed.epub +0 -0
  50. bindery_cli-0.31.0/test_facility/leaves_of_grass_tags_fixed.epub +0 -0
  51. bindery_cli-0.31.0/test_facility/muse_of_nightmares.epub +0 -0
  52. bindery_cli-0.31.0/test_facility/muse_of_nightmares_fixed.epub +0 -0
  53. bindery_cli-0.31.0/test_facility/poor_mans_fight.epub +0 -0
  54. bindery_cli-0.31.0/test_facility/poor_mans_fight_fixed.epub +0 -0
  55. bindery_cli-0.31.0/test_facility/test_block_in_inline.py +0 -69
  56. bindery_cli-0.31.0/test_facility/test_duplicate_playorder.py +0 -63
  57. bindery_cli-0.31.0/test_facility/test_empty_body.py +0 -59
  58. bindery_cli-0.31.0/test_facility/test_illegal_tags.py +0 -64
  59. bindery_cli-0.31.0/test_facility/test_invalid_ids.py +0 -79
  60. bindery_cli-0.31.0/test_facility/test_invalid_value.py +0 -65
  61. bindery_cli-0.31.0/test_facility/test_missing_title.py +0 -80
  62. bindery_cli-0.31.0/test_facility/unworthy_gods.epub +0 -0
  63. bindery_cli-0.31.0/test_facility/unworthy_gods_fixed.epub +0 -0
  64. bindery_cli-0.31.0/tests/test_validate.py +0 -150
  65. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/.github/workflows/ci.yml +0 -0
  66. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/.github/workflows/publish.yml +0 -0
  67. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/AGENTS.md +0 -0
  68. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/LICENSE +0 -0
  69. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/docs/screenshots/library-sweep.png +0 -0
  70. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/logo.svg +0 -0
  71. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/scripts/FastSweep.java +0 -0
  72. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/scripts/fast_sweep.py +0 -0
  73. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/src/bindery/__main__.py +0 -0
  74. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/testing_facility/books_to_fix/.gitkeep +0 -0
  75. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/testing_facility/top500candidates/REPORT.md +0 -0
  76. {bindery_cli-0.31.0 → bindery_cli-0.33.0}/tests/test_version.py +0 -0
@@ -12,6 +12,11 @@ testing_facility/books_to_fix/*
12
12
  !testing_facility/books_to_fix/.gitkeep
13
13
  testing_facility/top500candidates/*.epub
14
14
 
15
+ # test_facility EPUBs stopped being tracked in v0.33.0 (forward-only; 9 of
16
+ # the 10 books are commercial and one untracked leak on a public repo is
17
+ # one too many). Files stay on disk, out of git.
18
+ test_facility/*.epub
19
+
15
20
  # Compiled Java scratch (FastSweep harness)
16
21
  *.class
17
22
 
@@ -34,14 +34,29 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
34
34
  minimal content. v0.14–v0.16 ran these unconditionally, which broke this rule;
35
35
  v0.17.0 restored it. `--unwrap-illegal-tags` additionally protects any illegal-tag
36
36
  name that an EPUB stylesheet styles as an element selector (`css_protected_tags`,
37
- book-wide, inline `<style>` blocks included).
37
+ book-wide, inline `<style>` blocks included). The two EPUB2-targeted fixes
38
+ (`--strip-epub3-attrs`, `--downgrade-epub3-tags`) are additionally gated on the
39
+ package version in the OPF (`package_version()`): inert on EPUB 3 books and when
40
+ no version can be read, since their target defects only exist below EPUB 3; the
41
+ attribute scrub is anchored to real start tags via
42
+ `transforms.strip_attrs_in_start_tags` (prose mentioning the attributes and
43
+ CDATA/comment content are preserved), and `strip_invalid_value` matches the
44
+ attribute name with a `(?<![\w:.-])value` lookbehind so `data-value` survives.
38
45
  * **Lossy strips** (`--strip-pagination`, `--strip-broken-tags`, `--strip-watermarks`;
39
46
  pagination.py, watermark.py): remove only what a converter injected (page numbers,
40
47
  running headers, leaked tags, watermarks), fenced behind character conservation, tag
41
- balance, and the epubcheck no-regression bar.
48
+ balance, and the epubcheck no-regression bar. The watermark anchored pass's
49
+ whole-match fallback only fires when the match holds nothing but the stamp; a
50
+ larger match (an unclosed stamp anchor that swallowed prose) is refused, counted
51
+ in `RepairReport.watermark_refusals`, and surfaced as a `manual_watermark_repair`
52
+ decision by `run phase1` on both the read-only and apply paths.
42
53
  Do not let any NEW fix touch content without its own flag; if a candidate repair cannot
43
54
  be made deterministically safe, it does not belong here — report it for manual repair
44
- instead.
55
+ instead. A fourth, smaller group sits beside the structural repairs and lossy strips:
56
+ the safe opt-ins (`--fix-ids`, `--add-img-alt`, `--strip-bad-attrs`,
57
+ `--escape-unknown-entities`) plus `--encode-url-spaces`, which repair without
58
+ altering visible markup; the "twelve structural repairs and three lossy strips"
59
+ phrasing elsewhere in these docs was never the complete inventory.
45
60
  - **The gate is the safety contract.** Never apply a repair epubcheck has not accepted.
46
61
  Respect the two-mode logic in `validate.gate` (fatal-fixing tolerates error unmasking;
47
62
  error-cleanup does not). The lossy strips (`--strip-pagination`, `--strip-broken-tags`,
@@ -58,14 +73,14 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
58
73
 
59
74
  ## Layout
60
75
 
61
- - `src/bindery/transforms.py`: pure `str -> (str, int)` text transforms (including `strip_broken_tags`).
76
+ - `src/bindery/transforms.py`: pure `str -> (str, int)` text transforms (including `strip_broken_tags`). Since v0.32.0: `strip_attrs_in_start_tags` anchors attribute edits on the quote-aware start-tag matcher, `_outside_protected` forwards arguments (context-carrying transforms can be decorated), `fix_id_colons` matches only the bare `id` attribute and only internal fragments (`fix_ncx_src_fragments` carries renames into the NCX), `fix_ncx_playorder` is anchored to `<navPoint>` start tags, and the css selector boundary covers namespaced and functional selector forms.
62
77
  - `src/bindery/pagination.py`: the opt-in lossy page-number strip (runhead detection, page-layer decision, block-centric removal/merge, safety nets).
63
78
  - `src/bindery/watermark.py`: the opt-in lossy watermark strip (anchored and anchorless signature removal).
64
79
  - `src/bindery/reserialize.py`: structural repair via `html5lib`.
65
80
  - `src/bindery/audit.py`: read-only body-text audits (`content`, `pagenumbers`, `emptytext`, `ocr`, and `monolithic` since v0.21.0, `--max-doc-chars N`) behind the `audit` subcommand (v0.15.0). Since v0.18.0 library mode resolves EPUB paths through `cquarry.get_format_path()` and can apply a tag to flagged books via `cquarry.write.WritableCalibreDB` (only with explicit `--tag`; the only sanctioned write path). Since v0.19.0 `audit --id BOOK_ID` audits a single library book through cquarry's single-entity `get_book()` fetch (no library-wide cache; supports `--tag`; incompatible with the directory argument; comma lists since v0.23.0). Since v0.22.0 every archive entry is fully read (CRC + decompression) before analysis — a damaged entry reports its own CORRUPT verdict instead of feeding emptytext — and manifest/NCX references to absent files classify as `convention` (bloated ToC, consecutive span) or `fragment` (broken span). Hazard: the scan loop reuses `tag` as its per-book display column, so the `--tag` argument is captured as `audit_tag` at function entry — do not collapse them again. Since v0.29.0 `audit --json FILE` writes per-file analyzer verdicts in the `library --json` shape (directory, library, and single-book modes; with `--id`, exactly one id; emptytext omitted from a record when the archive verdict owns the body-text story).
66
81
  - `src/bindery/epub.py`: archive rewrite, NCX uid sync, RepairReport, mismatch detection, and the opt-in structural-repair plumbing (including the CSS precondition scan).
67
82
  - `src/bindery/validate.py`: epubcheck wrapper, the `gate` (improvement) and `no_worse` (no-regression, for the lossy strips) acceptance bars.
68
- - `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching) — the `(id)` directory-name regex is only the no-catalog fallback. Since v0.24.0 `install_format()` places the repaired file atomically and updates the `data` row through `cquarry.write.WritableCalibreDB` (remove+add in one `batch()`; `add_format` refuses duplicates); the external `calibredb` CLI is gone. Never reintroduce directory-name guessing as the primary source: renamed/mismatched directories would replace the wrong book.
83
+ - `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching) — the `(id)` directory-name regex is only the no-catalog fallback. Since v0.24.0 `install_format()` places the repaired file atomically and updates the `data` row through `cquarry.write.WritableCalibreDB` (since v0.31.0 via `set_format`, cquarry 1.17's sanctioned remove+add in one transaction); the external `calibredb` CLI is gone. Since v0.32.0 a guessed id drives a row update only when metadata.db corroborates it (`_verify_guess`: books row exists, file in the book's own directory, catalogued `data.name` when a row exists); anything else saves in place and leaves the catalog untouched — never reintroduce directory-name guessing as the primary source: renamed/mismatched directories would replace the wrong book.
69
84
  - `src/bindery/cli.py`: `repair` and `library` subcommands, including `--all` and `--install-to-calibre`; since v0.29.0 also the `run` verbs: `phase1` (pre-import vetting over loose files: the audit battery composed with one gated repair sweep; read-only until `--apply-lossy`, which IS the lossy-strip consent) and `phase3` (the scoped post-import apply step: `library --id --sweep --only all --apply --all --install-to-calibre` plus a pre/post summary; mechanically refuses unscoped sweeps with exit 2). Both take `--non-interactive` and surface open questions as `decisions_needed` in their JSON, never prompts. They drive the shipped `run_library` through the real parser, so no flag can drift between the verb and the subcommand it wraps. `library --workers N` (v0.30.0) parallelizes only the sweep's candidate pass, in windows of N consumed in order so `--limit` stays lazy; the repair phase is never parallel (shared workdir, atomic-replacement contract).
70
85
  - `tests/`: transforms, end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic, archive integrity, spine classification), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, and the version pin.
71
86
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: bindery-cli
3
- Version: 0.31.0
3
+ Version: 0.33.0
4
4
  Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
5
5
  Author: vrnvctss
6
6
  License: MIT
@@ -51,14 +51,14 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
51
51
  - **`--reserialize`**: rebuild content documents that are still malformed by re-parsing them with html5lib and re-emitting XHTML, closing unclosed `<p>`/`<div>`/`<span>`/`<blockquote>` that the regex transforms cannot. Runs only on documents that are not already well-formed, so good files are untouched.
52
52
  - **`--strip-bad-attrs`**: drop attributes that are invalid XML (a name starting with a digit, or a namespaced name whose prefix is never declared, like Office VML `v:shapes`). Surgical and a no-op on well-formed files.
53
53
  - **`--escape-unknown-entities`**: escape entity names that are not in the HTML5 table (`&foo;` becomes `&amp;foo;`), which renders exactly as browsers already render an unknown entity. Documents whose DOCTYPE carries an internal subset are skipped wholesale, since a subset can declare custom entities.
54
- - **`--fix-id-colons`**: translates illegal colons in `id="X:Y"` and their matching `#X:Y` fragment references to valid underscores.
54
+ - **`--fix-id-colons`**: translates illegal colons in `id="X:Y"` and their matching internal `#X:Y` fragment references to valid underscores. External URL fragments are left verbatim (they name positions in other documents), `data-id` values are never touched, and the NCX's `content src` fragments follow the rename so the table of contents never dangles against the ids it references.
55
55
  - **`--fix-empty-body`**: appends a non-breaking space `&nbsp;` to strictly empty body tags to satisfy parser requirements.
56
56
  - **`--fix-missing-title`**: injects a `<title>Unknown</title>` fallback in the `<head>` if missing, handling both empty `<title/>` self-closing tags and entirely absent tags.
57
57
  - **`--unwrap-block-in-inline`**: safely unwraps `<span>` tags that illegally contain a block-level element (e.g. `<div>` or `<p>`), leaving the block element intact.
58
58
  - **`--strip-invalid-value`**: systematically strips invalid `value="..."` attributes from elements like `<div>`, `<span>`, `<p>`, etc.
59
59
  - **`--fix-page-map`**: normalizes legacy page-map markup — drops the non-standard `page-map="..."` attribute from the OPF `<spine>` and adds `class="pages"` to classless NCX `<pageList>` elements (epubcheck rejects both on older HarperCollins / Anna's Archive conversions).
60
- - **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package — `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive).
61
- - **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents — `figure`/`section` to `div`, `figcaption` to `p` — keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed.
60
+ - **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package — `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive). The edit is anchored to real start tags, so prose that merely mentions `epub:type="chapter"` is preserved.
61
+ - **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents — `figure`/`section` to `div`, `figcaption` to `p` — keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed. Both EPUB2-targeted fixes are gated on the package version declared in the OPF: on an EPUB 3 book they do nothing, because their target defects only exist below EPUB 3.
62
62
  - **`--unwrap-illegal-tags`**: strips completely invalid or deprecated HTML tags that break EPUB3 validation (`<st>`, `<sentence>`, `<o>`, `<w>`, `<pagebreak>`) while retaining their inner text. Any of those names styled as an *element selector* by an EPUB stylesheet (`.css` entries and inline `<style>` blocks alike; class/id selectors like `.st`/`#w` don't count) is protected for the whole book, guaranteeing format preservation.
63
63
  - **`--prune-missing-resources`**: removes references to files the archive does not contain (RSC-007/PKG-010): dead `<link>` elements, anchors' `href` to absent files (anchor text preserved), absent `<img>` sources (replaced by their alt text when they carry one), and orphaned non-spine OPF manifest items. Spine documents are never pruned: a missing spine document is reported as a damaged fragment, not silently dropped.
64
64
  - **`--strip-broken-anchors`**: strips href attributes that cannot resolve, keeping the anchor text byte-for-byte: a `#fragment` the target document does not define (RSC-020/RSC-012; NCX navTargets fall back to the document target, so chapter navigation survives) and unresolvable URI schemes (`kindle:`, `file:`).
@@ -67,9 +67,9 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
67
67
 
68
68
  Three opt-in fixes are **lossy** and stand apart from the semantics-preserving rest:
69
69
 
70
- - **`--strip-pagination`**: remove print page numbers and running headers that a PDF/OCR conversion baked into the body text as literal paragraphs (so they reflow into the middle of a sentence: "where the hay cart **16** was taking him"). It removes only that injected furniture, never the author's prose: where a number split a sentence it rejoins the two paragraphs (closing up a word split like `compli-` / `mentary`), and it preserves roman chapter numbers, page-list nav anchors, and years. A book is only treated as paginated when it has both a dense run of arabic numbers and several confident mid-sentence interrupts, so a merely chapter-numbered book is left alone. Three safety nets guard every edit (character conservation, tag balance, and an epubcheck no-regression check); any failure leaves the document untouched.
70
+ - **`--strip-pagination`**: remove print page numbers and running headers that a PDF/OCR conversion baked into the body text as literal paragraphs (so they reflow into the middle of a sentence: "where the hay cart **16** was taking him"). It removes only that injected furniture, never the author's prose: where a number split a sentence it rejoins the two paragraphs (closing up a word split like `compli-` / `mentary`), and it preserves roman chapter numbers, page-list nav anchors, and years. A roman numeral counts as a page number only when it is well-formed and under 100, so ordinary words (`mid`, `mix`, `civil`) are never mistaken for pages. A book is only treated as paginated when it has both a dense run of arabic numbers and several confident mid-sentence interrupts, so a merely chapter-numbered book is left alone. Three safety nets guard every edit (character conservation, tag balance, and an epubcheck no-regression check); any failure leaves the document untouched.
71
71
  - **`--strip-broken-tags`**: remove leaked HTML closing tags missing their open brackets (e.g. `</p>`) that render as raw text.
72
- - **`--strip-watermarks`**: remove known producer and distributor watermarks (e.g. OceanofPDF, ABC Amber LIT Converter) and stray marker files. It locates the stamp and deletes the outermost wrapper whose entire visible text is the watermark, ensuring prose that merely mentions the URL is preserved.
72
+ - **`--strip-watermarks`**: remove known producer and distributor watermarks (e.g. OceanofPDF, ABC Amber LIT Converter) and stray marker files. It locates the stamp and deletes the outermost wrapper whose entire visible text is the watermark, ensuring prose that merely mentions the URL is preserved. An inline stamp link is deleted only when it holds nothing but the stamp; a match too large to be safe (an unclosed stamp anchor that swallowed prose) is refused and reported for manual repair rather than deleted.
73
73
 
74
74
  All lossy edits are invisible to epubcheck, so they are accepted when the result is *no worse* rather than measurably better.
75
75
 
@@ -82,6 +82,8 @@ Every repair is gated by [epubcheck](https://github.com/w3c/epubcheck). The acce
82
82
 
83
83
  Introducing a net-new fatal is always rejected. If epubcheck itself fails to run (crash, timeout, unparsable output), the book is reported as an error and never applied; only an explicit `--no-validate` skips the gate. Originals are never modified except by an explicit, atomic in-place replace (see below), and even then only after the gate accepts the result.
84
84
 
85
+ Under the hood, the oracle can run two ways: the usual `epubcheck --json -` subprocess, or a small in-process daemon that compiles `FastDaemon.java` into a temp directory at runtime and drives a warm JVM over a pipe. The daemon is a pool bounded by `--workers` (serial runs use one), its every roundtrip is bounded by the caller's timeout, and any failure tears it down and falls back to the subprocess for good — the two paths must never disagree about a book, and the subprocess oracle is the fallback by design. The daemon is also the scariest under-tested code in the repo and depends on a matched javac/java pair; its status is recorded in the roadmap's Phase 12 postscript and Phase 14 backlog.
86
+
85
87
  The lossy modes (`--strip-pagination`, `--strip-broken-tags`, and `--strip-watermarks`) are the exception to the "must improve" rule. Since they remove visible markup rather than correcting XML schema violations, epubcheck counts often remain unchanged. They are accepted when the result is **no worse** (no net-new fatals or errors), relying on strict programmatic safety nets instead.
86
88
 
87
89
  ## Install
@@ -192,19 +194,22 @@ Apply all safe and lossy repairs directly into the Calibre database natively:
192
194
  bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-calibre
193
195
  ```
194
196
 
195
- - `--only {fatals,ncx,all}` restricts the candidate set. `ncx` targets NCX-001 mismatches (detected without epubcheck); `fatals` needs `--audit`.
197
+ - `--only {fatals,ncx,all}` restricts the candidate set. `ncx` targets NCX-001 mismatches (detected without epubcheck); `fatals` needs `--audit` or `--sweep`.
196
198
  - `--id <ids>` limits the sweep to a comma-separated list of Calibre book IDs, skipping the full library walk.
197
199
  - `--audit CSV` (the `fatals,errors,warnings,path` format produced by an epubcheck sweep) skips clean books so a run is fast. Paths are resolved on both sides, and a CSV that matches nothing triggers a loud warning instead of silently selecting zero books.
198
- - `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice. bindery-cli first tries a persistent Java daemon (`EpubcheckDaemon`) to eliminate the per-book JVM startup; in practice the daemon cannot boot on most installs (its classpath carries only epubcheck's launcher jar, not the `lib/` dependencies epubcheck itself puts on the classpath), and the run silently falls back to the standard per-book epubcheck subprocess at the usual seconds per book. Budget accordingly. Fixing the classpath is a recorded decision (it would also change the counts the gate measures), not a shipped behavior. Combine with `--only fatals` for a self-contained "find and fix the broken books" run.
200
+ - `--limit N` processes at most N candidates and stops opening archives after them (lazy within one worker window). A value below 1 is a usage error.
201
+ - `--min-chars N` and `--thin-chars N` are the audit's EMPTY and THIN body-text thresholds (defaults 2000 and 20000); `--min-chars` above `--thin-chars` is a usage error. `--max-doc-chars N` is the audit monolithic-document flag threshold (default 300,000).
202
+ - `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice. bindery-cli may serve checks from a small persistent Java daemon (see the gate section above for how it is bounded and how it fails safe); wherever the daemon cannot serve, the run falls back to the standard per-book epubcheck subprocess at the usual seconds per book. Combine with `--only fatals` for a self-contained "find and fix the broken books" run.
199
203
  - `--workers N` runs the sweep's candidate pass through N concurrent epubcheck workers (default 1: serial, unchanged). The subprocess releases the GIL, so threads parallelize the oracle honestly; books are checked in windows of N consumed in order, so the candidate set matches the serial sweep and `--limit` stays lazy within one window of overshoot. The repair phase stays serial: that is where the shared workdir and the atomic-replacement contract live.
200
204
  - `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
201
205
  - `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
202
206
  - `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
203
- - `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied` — Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning.
207
+ - `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied` — Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning. A directory-name-guessed id is trusted only when `metadata.db` corroborates it (the book exists and the file is that book's catalogued EPUB); an uncatalogued stray file inside a book directory is saved in place and the catalog row is left untouched.
204
208
  - `--all` automatically turns on all opt-in non-fatal fixes and lossy strips (pagination, watermarks, bad attributes, unknown entities, image alt tags, etc.) in a single run.
205
209
  - Only the `.epub` is replaced. `metadata.opf`, `cover.jpg`, and `metadata.db` are left for Calibre's Quality Check sync to reconcile.
206
210
  - A per-book progress line goes to stderr (stdout stays a clean report); `--quiet` suppresses it. A corrupt or unreadable book is reported and skipped, never aborting the sweep.
207
- - Exit codes: 0 for a clean sweep, 1 for a usage error, 2 when any book was rejected, unreadable, or failed epubcheck (for scripts and cron).
211
+ - Exit codes: 0 for a clean sweep, 1 for a usage error, 2 when any book was rejected, unreadable, or failed epubcheck (for scripts and cron). Note that argparse-level misuse (an unknown flag or a malformed argument) exits with 2 before any validation runs; the tool's own usage validations exit 1.
212
+ - A `partial` book (improved but still unable to open) is reported for manual follow-up and does not fail a `library` run; `run phase1` counts it as trouble and exits 2.
208
213
  - `repair` refuses to overwrite an existing output file unless `--force` is given.
209
214
 
210
215
  ## Run slices
@@ -246,15 +251,22 @@ cd ~/docs/Calibre\ Library && bindery run phase3 --ids 5071,5072 --json phase3-r
246
251
 
247
252
  `scripts/` holds standalone, read-only utilities that are useful for EPUB maintenance but fall outside bindery-cli's repair contract (fixing what they find would be a content change, which bindery-cli makes only via the opt-in `--strip-pagination`):
248
253
 
249
- - `find_missing_images.py`: scans a library tree and reports every book whose `<img>` tags point at files that do not exist inside the archive (a common defect in converted EPUBs). Reads the archives in place; nothing is unpacked or written. The library path is set at the bottom of the script.
250
254
  - `fast_sweep.py`: compiles and drives `FastSweep.java`, a parallel JVM epubcheck sweep that pays JVM startup once instead of per book (a 7,000-book dry-run sweep drops from hours to minutes). `--mode=audit` emits the `fatals,errors,warnings,path` CSV that `bindery library --audit` reads; `--mode=extract` emits per-book error codes for aggregation (`--summary` prints the per-code report). The epubcheck jar is located from the `epubcheck` launcher or `EPUBCHECK_JAR`; the harness compiles once (`--release 25`) and is cached on mtime.
251
255
 
256
+ The older `find_*.py` detection wedges and the `sweep.sh`/`FastSweepExtract.java` prototypes were removed: every detection they performed is now a shipped, tested flag (missing images are `--prune-missing-resources`, and the rest of the RSC-005/PKG-010 family shipped through Phases 6-12), and keeping two copies meant future behavior changes landed in only one of them. Git history preserves them.
257
+
252
258
  ## Development
253
259
 
254
260
  ```sh
255
- ./run_tests.sh # unittest suite
261
+ ./run_tests.sh # ruff (CI's pin) + unittest suite
256
262
  ```
257
263
 
264
+ The suite builds its own synthetic EPUBs in temporary directories. `testing_facility/`
265
+ holds staged real-book copies for manual dry runs (gitignored); `test_facility/` holds
266
+ loose local books that stay on disk but out of git (untracked since v0.33.0,
267
+ forward-only, because 9 of the 10 are commercial and one untracked leak on a public
268
+ repo is one too many).
269
+
258
270
  See [spec.md](spec.md) for the full contract and [roadmap.md](roadmap.md) for what is planned.
259
271
 
260
272
  ## License
@@ -29,14 +29,14 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
29
29
  - **`--reserialize`**: rebuild content documents that are still malformed by re-parsing them with html5lib and re-emitting XHTML, closing unclosed `<p>`/`<div>`/`<span>`/`<blockquote>` that the regex transforms cannot. Runs only on documents that are not already well-formed, so good files are untouched.
30
30
  - **`--strip-bad-attrs`**: drop attributes that are invalid XML (a name starting with a digit, or a namespaced name whose prefix is never declared, like Office VML `v:shapes`). Surgical and a no-op on well-formed files.
31
31
  - **`--escape-unknown-entities`**: escape entity names that are not in the HTML5 table (`&foo;` becomes `&amp;foo;`), which renders exactly as browsers already render an unknown entity. Documents whose DOCTYPE carries an internal subset are skipped wholesale, since a subset can declare custom entities.
32
- - **`--fix-id-colons`**: translates illegal colons in `id="X:Y"` and their matching `#X:Y` fragment references to valid underscores.
32
+ - **`--fix-id-colons`**: translates illegal colons in `id="X:Y"` and their matching internal `#X:Y` fragment references to valid underscores. External URL fragments are left verbatim (they name positions in other documents), `data-id` values are never touched, and the NCX's `content src` fragments follow the rename so the table of contents never dangles against the ids it references.
33
33
  - **`--fix-empty-body`**: appends a non-breaking space `&nbsp;` to strictly empty body tags to satisfy parser requirements.
34
34
  - **`--fix-missing-title`**: injects a `<title>Unknown</title>` fallback in the `<head>` if missing, handling both empty `<title/>` self-closing tags and entirely absent tags.
35
35
  - **`--unwrap-block-in-inline`**: safely unwraps `<span>` tags that illegally contain a block-level element (e.g. `<div>` or `<p>`), leaving the block element intact.
36
36
  - **`--strip-invalid-value`**: systematically strips invalid `value="..."` attributes from elements like `<div>`, `<span>`, `<p>`, etc.
37
37
  - **`--fix-page-map`**: normalizes legacy page-map markup — drops the non-standard `page-map="..."` attribute from the OPF `<spine>` and adds `class="pages"` to classless NCX `<pageList>` elements (epubcheck rejects both on older HarperCollins / Anna's Archive conversions).
38
- - **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package — `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive).
39
- - **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents — `figure`/`section` to `div`, `figcaption` to `p` — keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed.
38
+ - **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package — `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive). The edit is anchored to real start tags, so prose that merely mentions `epub:type="chapter"` is preserved.
39
+ - **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents — `figure`/`section` to `div`, `figcaption` to `p` — keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed. Both EPUB2-targeted fixes are gated on the package version declared in the OPF: on an EPUB 3 book they do nothing, because their target defects only exist below EPUB 3.
40
40
  - **`--unwrap-illegal-tags`**: strips completely invalid or deprecated HTML tags that break EPUB3 validation (`<st>`, `<sentence>`, `<o>`, `<w>`, `<pagebreak>`) while retaining their inner text. Any of those names styled as an *element selector* by an EPUB stylesheet (`.css` entries and inline `<style>` blocks alike; class/id selectors like `.st`/`#w` don't count) is protected for the whole book, guaranteeing format preservation.
41
41
  - **`--prune-missing-resources`**: removes references to files the archive does not contain (RSC-007/PKG-010): dead `<link>` elements, anchors' `href` to absent files (anchor text preserved), absent `<img>` sources (replaced by their alt text when they carry one), and orphaned non-spine OPF manifest items. Spine documents are never pruned: a missing spine document is reported as a damaged fragment, not silently dropped.
42
42
  - **`--strip-broken-anchors`**: strips href attributes that cannot resolve, keeping the anchor text byte-for-byte: a `#fragment` the target document does not define (RSC-020/RSC-012; NCX navTargets fall back to the document target, so chapter navigation survives) and unresolvable URI schemes (`kindle:`, `file:`).
@@ -45,9 +45,9 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
45
45
 
46
46
  Three opt-in fixes are **lossy** and stand apart from the semantics-preserving rest:
47
47
 
48
- - **`--strip-pagination`**: remove print page numbers and running headers that a PDF/OCR conversion baked into the body text as literal paragraphs (so they reflow into the middle of a sentence: "where the hay cart **16** was taking him"). It removes only that injected furniture, never the author's prose: where a number split a sentence it rejoins the two paragraphs (closing up a word split like `compli-` / `mentary`), and it preserves roman chapter numbers, page-list nav anchors, and years. A book is only treated as paginated when it has both a dense run of arabic numbers and several confident mid-sentence interrupts, so a merely chapter-numbered book is left alone. Three safety nets guard every edit (character conservation, tag balance, and an epubcheck no-regression check); any failure leaves the document untouched.
48
+ - **`--strip-pagination`**: remove print page numbers and running headers that a PDF/OCR conversion baked into the body text as literal paragraphs (so they reflow into the middle of a sentence: "where the hay cart **16** was taking him"). It removes only that injected furniture, never the author's prose: where a number split a sentence it rejoins the two paragraphs (closing up a word split like `compli-` / `mentary`), and it preserves roman chapter numbers, page-list nav anchors, and years. A roman numeral counts as a page number only when it is well-formed and under 100, so ordinary words (`mid`, `mix`, `civil`) are never mistaken for pages. A book is only treated as paginated when it has both a dense run of arabic numbers and several confident mid-sentence interrupts, so a merely chapter-numbered book is left alone. Three safety nets guard every edit (character conservation, tag balance, and an epubcheck no-regression check); any failure leaves the document untouched.
49
49
  - **`--strip-broken-tags`**: remove leaked HTML closing tags missing their open brackets (e.g. `</p>`) that render as raw text.
50
- - **`--strip-watermarks`**: remove known producer and distributor watermarks (e.g. OceanofPDF, ABC Amber LIT Converter) and stray marker files. It locates the stamp and deletes the outermost wrapper whose entire visible text is the watermark, ensuring prose that merely mentions the URL is preserved.
50
+ - **`--strip-watermarks`**: remove known producer and distributor watermarks (e.g. OceanofPDF, ABC Amber LIT Converter) and stray marker files. It locates the stamp and deletes the outermost wrapper whose entire visible text is the watermark, ensuring prose that merely mentions the URL is preserved. An inline stamp link is deleted only when it holds nothing but the stamp; a match too large to be safe (an unclosed stamp anchor that swallowed prose) is refused and reported for manual repair rather than deleted.
51
51
 
52
52
  All lossy edits are invisible to epubcheck, so they are accepted when the result is *no worse* rather than measurably better.
53
53
 
@@ -60,6 +60,8 @@ Every repair is gated by [epubcheck](https://github.com/w3c/epubcheck). The acce
60
60
 
61
61
  Introducing a net-new fatal is always rejected. If epubcheck itself fails to run (crash, timeout, unparsable output), the book is reported as an error and never applied; only an explicit `--no-validate` skips the gate. Originals are never modified except by an explicit, atomic in-place replace (see below), and even then only after the gate accepts the result.
62
62
 
63
+ Under the hood, the oracle can run two ways: the usual `epubcheck --json -` subprocess, or a small in-process daemon that compiles `FastDaemon.java` into a temp directory at runtime and drives a warm JVM over a pipe. The daemon is a pool bounded by `--workers` (serial runs use one), its every roundtrip is bounded by the caller's timeout, and any failure tears it down and falls back to the subprocess for good — the two paths must never disagree about a book, and the subprocess oracle is the fallback by design. The daemon is also the scariest under-tested code in the repo and depends on a matched javac/java pair; its status is recorded in the roadmap's Phase 12 postscript and Phase 14 backlog.
64
+
63
65
  The lossy modes (`--strip-pagination`, `--strip-broken-tags`, and `--strip-watermarks`) are the exception to the "must improve" rule. Since they remove visible markup rather than correcting XML schema violations, epubcheck counts often remain unchanged. They are accepted when the result is **no worse** (no net-new fatals or errors), relying on strict programmatic safety nets instead.
64
66
 
65
67
  ## Install
@@ -170,19 +172,22 @@ Apply all safe and lossy repairs directly into the Calibre database natively:
170
172
  bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-calibre
171
173
  ```
172
174
 
173
- - `--only {fatals,ncx,all}` restricts the candidate set. `ncx` targets NCX-001 mismatches (detected without epubcheck); `fatals` needs `--audit`.
175
+ - `--only {fatals,ncx,all}` restricts the candidate set. `ncx` targets NCX-001 mismatches (detected without epubcheck); `fatals` needs `--audit` or `--sweep`.
174
176
  - `--id <ids>` limits the sweep to a comma-separated list of Calibre book IDs, skipping the full library walk.
175
177
  - `--audit CSV` (the `fatals,errors,warnings,path` format produced by an epubcheck sweep) skips clean books so a run is fast. Paths are resolved on both sides, and a CSV that matches nothing triggers a loud warning instead of silently selecting zero books.
176
- - `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice. bindery-cli first tries a persistent Java daemon (`EpubcheckDaemon`) to eliminate the per-book JVM startup; in practice the daemon cannot boot on most installs (its classpath carries only epubcheck's launcher jar, not the `lib/` dependencies epubcheck itself puts on the classpath), and the run silently falls back to the standard per-book epubcheck subprocess at the usual seconds per book. Budget accordingly. Fixing the classpath is a recorded decision (it would also change the counts the gate measures), not a shipped behavior. Combine with `--only fatals` for a self-contained "find and fix the broken books" run.
178
+ - `--limit N` processes at most N candidates and stops opening archives after them (lazy within one worker window). A value below 1 is a usage error.
179
+ - `--min-chars N` and `--thin-chars N` are the audit's EMPTY and THIN body-text thresholds (defaults 2000 and 20000); `--min-chars` above `--thin-chars` is a usage error. `--max-doc-chars N` is the audit monolithic-document flag threshold (default 300,000).
180
+ - `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice. bindery-cli may serve checks from a small persistent Java daemon (see the gate section above for how it is bounded and how it fails safe); wherever the daemon cannot serve, the run falls back to the standard per-book epubcheck subprocess at the usual seconds per book. Combine with `--only fatals` for a self-contained "find and fix the broken books" run.
177
181
  - `--workers N` runs the sweep's candidate pass through N concurrent epubcheck workers (default 1: serial, unchanged). The subprocess releases the GIL, so threads parallelize the oracle honestly; books are checked in windows of N consumed in order, so the candidate set matches the serial sweep and `--limit` stays lazy within one window of overshoot. The repair phase stays serial: that is where the shared workdir and the atomic-replacement contract live.
178
182
  - `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
179
183
  - `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
180
184
  - `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
181
- - `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied` — Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning.
185
+ - `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied` — Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning. A directory-name-guessed id is trusted only when `metadata.db` corroborates it (the book exists and the file is that book's catalogued EPUB); an uncatalogued stray file inside a book directory is saved in place and the catalog row is left untouched.
182
186
  - `--all` automatically turns on all opt-in non-fatal fixes and lossy strips (pagination, watermarks, bad attributes, unknown entities, image alt tags, etc.) in a single run.
183
187
  - Only the `.epub` is replaced. `metadata.opf`, `cover.jpg`, and `metadata.db` are left for Calibre's Quality Check sync to reconcile.
184
188
  - A per-book progress line goes to stderr (stdout stays a clean report); `--quiet` suppresses it. A corrupt or unreadable book is reported and skipped, never aborting the sweep.
185
- - Exit codes: 0 for a clean sweep, 1 for a usage error, 2 when any book was rejected, unreadable, or failed epubcheck (for scripts and cron).
189
+ - Exit codes: 0 for a clean sweep, 1 for a usage error, 2 when any book was rejected, unreadable, or failed epubcheck (for scripts and cron). Note that argparse-level misuse (an unknown flag or a malformed argument) exits with 2 before any validation runs; the tool's own usage validations exit 1.
190
+ - A `partial` book (improved but still unable to open) is reported for manual follow-up and does not fail a `library` run; `run phase1` counts it as trouble and exits 2.
186
191
  - `repair` refuses to overwrite an existing output file unless `--force` is given.
187
192
 
188
193
  ## Run slices
@@ -224,15 +229,22 @@ cd ~/docs/Calibre\ Library && bindery run phase3 --ids 5071,5072 --json phase3-r
224
229
 
225
230
  `scripts/` holds standalone, read-only utilities that are useful for EPUB maintenance but fall outside bindery-cli's repair contract (fixing what they find would be a content change, which bindery-cli makes only via the opt-in `--strip-pagination`):
226
231
 
227
- - `find_missing_images.py`: scans a library tree and reports every book whose `<img>` tags point at files that do not exist inside the archive (a common defect in converted EPUBs). Reads the archives in place; nothing is unpacked or written. The library path is set at the bottom of the script.
228
232
  - `fast_sweep.py`: compiles and drives `FastSweep.java`, a parallel JVM epubcheck sweep that pays JVM startup once instead of per book (a 7,000-book dry-run sweep drops from hours to minutes). `--mode=audit` emits the `fatals,errors,warnings,path` CSV that `bindery library --audit` reads; `--mode=extract` emits per-book error codes for aggregation (`--summary` prints the per-code report). The epubcheck jar is located from the `epubcheck` launcher or `EPUBCHECK_JAR`; the harness compiles once (`--release 25`) and is cached on mtime.
229
233
 
234
+ The older `find_*.py` detection wedges and the `sweep.sh`/`FastSweepExtract.java` prototypes were removed: every detection they performed is now a shipped, tested flag (missing images are `--prune-missing-resources`, and the rest of the RSC-005/PKG-010 family shipped through Phases 6-12), and keeping two copies meant future behavior changes landed in only one of them. Git history preserves them.
235
+
230
236
  ## Development
231
237
 
232
238
  ```sh
233
- ./run_tests.sh # unittest suite
239
+ ./run_tests.sh # ruff (CI's pin) + unittest suite
234
240
  ```
235
241
 
242
+ The suite builds its own synthetic EPUBs in temporary directories. `testing_facility/`
243
+ holds staged real-book copies for manual dry runs (gitignored); `test_facility/` holds
244
+ loose local books that stay on disk but out of git (untracked since v0.33.0,
245
+ forward-only, because 9 of the 10 are commercial and one untracked leak on a public
246
+ repo is one too many).
247
+
236
248
  See [spec.md](spec.md) for the full contract and [roadmap.md](roadmap.md) for what is planned.
237
249
 
238
250
  ## License
@@ -1,4 +1,210 @@
1
1
  # bindery-cli Patch Notes
2
+ ## v0.33.0 (2026-09-10)
3
+
4
+ ### Phase 14 hardening: the apply path, the audit, and the safety contract's tests
5
+
6
+ The rest of the 2026-09-08 audit sweep's backlog, in the same shape as
7
+ v0.32.0: every fix confirmed against current code first, each with its
8
+ regression test.
9
+
10
+ **Apply-path and oracle safety**
11
+
12
+ - **A failed write can no longer abort a sweep raw.** The backup,
13
+ replace, and install calls sat bare in the loop, so an ENOSPC or
14
+ EACCES partway through a multi-hour run died with a traceback and no
15
+ record. Failures are now error Outcomes; the summary and JSON always
16
+ land, and exit 2 says scripts should look.
17
+ - **Backups rotate and stay out of the candidate set.** A second
18
+ `--apply` used to clobber the first backup (the only copy of the
19
+ author original); backups now rotate (`book.epub.bak2`, `.bak3`, ...).
20
+ A `--backup` directory inside the library root is refused (its copies
21
+ would be swept as candidates on the next run).
22
+ - **`--workers N` parallelism is real.** With epubcheck present, every
23
+ worker serialized behind the daemon's one pipe. The daemon is now a
24
+ bounded pool sized by `--workers` (default 1: the old behavior), a
25
+ busy pool falls back to subprocesses instead of blocking, every
26
+ roundtrip is bounded by the caller's timeout, and a daemon that dies
27
+ or wedges is torn down and never retried — one failed attempt exhausts
28
+ the pool so the sweep degrades to pure subprocess checks instead of
29
+ respawning a JVM per book. Measured: 8 books, 4 workers, two ~10s
30
+ waves. The evidence for the recorded daemon-classpath decision grew:
31
+ this machine's javac (27-ea) and java (25) cannot load each other's
32
+ classes, so the daemon has never served a count here, and when forced
33
+ to run its counts diverge from the CLI JSON oracle (243 vs 4 on one
34
+ book) — enabling it would change gate outcomes.
35
+ - **Smaller apply papercuts:** `--limit < 1` and a missing `--audit`
36
+ file are usage errors, not tracebacks; the fresh-format install
37
+ registers the catalogued file's own name instead of `repaired`; and
38
+ phase1 notes the dry-run `--backup` no-op instead of ignoring it.
39
+
40
+ **audit.py correctness**
41
+
42
+ - **ToC accounting reads inside the open zip,** a declared-but-absent
43
+ nav/NCX no longer poisons `Book.corrupt` (a healthy book with a
44
+ leftover `toc.ncx` entry is not a damaged archive), NCX `content src`
45
+ resolves against the NCX's own directory (a nested NCX counted every
46
+ target absent), and ToC hrefs are entity-decoded before comparison.
47
+ - **Archive and spine verdicts are first-class in library mode.** Their
48
+ sections were dead code in the library report loop: a CRC-corrupt book
49
+ printed `emptytext CLEAN` and exited 0 there while directory mode said
50
+ CORRUPT and exited 1. Both modes now agree.
51
+ - **The record pipeline's edges hold:** a failed `--json` write becomes a
52
+ trouble exit instead of a post-scan traceback; `--tag` skips
53
+ expected-foreign content findings; an unresolvable spine itemref is
54
+ counted into the emptytext detail instead of silently reading EMPTY;
55
+ DRM-encrypted entries get their own ENCRYPTED verdict
56
+ ("not repairable, skip") instead of CORRUPT re-source; and
57
+ part1a/part1b/part2-style spines no longer read as a broken span.
58
+ - **Dead surfaces removed** (a duplicate argparse CLI, an unreachable
59
+ analyzer, duplicated imports) and directory mode catches uppercase
60
+ `.EPUB` like the library walk always did.
61
+ - **Seven audit papercuts** closed: the block-stack pop now matches the
62
+ closing tag, the nav property matches whole tokens (`data-nav` is not
63
+ the nav), exit 3 is documented, the JSON analyzers list includes the
64
+ always-on verdicts, the console problem counter counts books like
65
+ `summary.problems`, `--min-chars` above `--thin-chars` is a usage
66
+ error, and duplicate archive entries are counted instead of resolving
67
+ last-wins in silence.
68
+
69
+ **Tests, hygiene, and docs**
70
+
71
+ - **Direct tests for the safety contract:** gate()/no_worse() including
72
+ the error-unmasking branch, the watermark gated-apply path,
73
+ atomic_replace failure injection (temp cleaned, target untouched,
74
+ re-raise), `--install-to-calibre` wiring, and library-mode `--tag` end
75
+ to end through cquarry's write path. Suite grew 342 → 381.
76
+ - **The weak two percent cleaned:** mid-file `__main__` blocks,
77
+ constant-assertion tests, and module aliases gone; the
78
+ non-interactive wiring is asserted through the real runner's JSON.
79
+ - **run_tests.sh runs CI's exact ruff pins** before the suite, so the
80
+ local loop fails where CI fails.
81
+ - **Repo hygiene:** the ten subsumed `scripts/find_*.py` wedges plus
82
+ `sweep.sh` and `FastSweepExtract.java` are retired (git keeps them);
83
+ the ten commercial EPUBs in `test_facility/` are untracked
84
+ (forward-only, files stay on disk) and the dead never-run suite there
85
+ is deleted.
86
+ - **Docs:** the daemon is admitted to exist (README gate section, spec
87
+ gate section) with its bounding and fail-safe fallback; README's
88
+ `--sweep` bullet stopped describing the fixed classpath bug as current;
89
+ the exit-code contract documents argparse's own exit 2 and the partial
90
+ book divergence; CLAUDE.md's exception taxonomy names the safe
91
+ opt-ins group.
92
+
93
+ **Recorded for Brandon (unchanged, evidence sharpened):** the daemon
94
+ classpath decision, the exit-code-from-partial-books decision (both
95
+ options written into its roadmap box; unifying on "trouble" is the
96
+ recommendation), and the spec.md reconciliation draft (the non-goals
97
+ vs. the shipped opt-ins, ready to apply on a go).
98
+
99
+ ## v0.32.0 (2026-09-10)
100
+
101
+ ### The four Bug Reports fixes (2026-09-08 sweep) and the anchored-attribute cluster
102
+
103
+ - **`--strip-watermarks` refuses anchored matches larger than the stamp**
104
+ (the sweep demonstrated two paragraphs of real prose vanishing with
105
+ count 1). The anchored stamp regex is DOTALL with no tag budget, so an
106
+ unclosed stamp `<a>` matched through real prose to the next unrelated
107
+ `</a>`, and the whole-match fallback deleted everything in between;
108
+ epubcheck cannot see deleted text, so the no-regression bar accepted the
109
+ loss. The fallback now fires only when the match demonstrably holds
110
+ nothing but the stamp (a tag-free body within stamp length, or visible
111
+ text exactly the watermark), a second `<a>` opening inside the match
112
+ refuses it outright, and anything larger stays in place, counted in the
113
+ new `RepairReport.watermark_refusals` and surfaced by `run phase1` as a
114
+ `manual_watermark_repair` decision on both the read-only and apply paths.
115
+ Safe shapes (pure wrappers, inline stamp-only links) are byte-identical
116
+ to before.
117
+ - **Roman page-number detection reads well-formed numerals only**, in both
118
+ `--strip-pagination` and the audit pagenumbers analyzer. The old
119
+ `[ivxlcdm]{2,7}` character set read ordinary English words as page
120
+ numbers (`mid` = 1499), and a standalone `<p>mid</p>` before a
121
+ lowercase-starting paragraph is exactly the confident interrupt shape,
122
+ so real words were deleted and merged away. A roman counts only when
123
+ well-formed (explicit subtractive pairs) and valued under 100, where
124
+ front-matter page numbers live and words do not (`mix` = M+IX = 1009).
125
+ Lowercase front-matter romans keep working; the audit copy's deliberately
126
+ different year handling is untouched.
127
+ - **`install_format` verifies directory-guessed ids against metadata.db.**
128
+ In directory mode, an uncatalogued stray `.epub` inside a book directory
129
+ made the `(id)` directory guess fire, and the row update re-registered
130
+ the catalogued format name with the stray file's size: the catalogued
131
+ 999-byte file untouched on disk while `data` recorded 8123 bytes. A
132
+ stale `(N)` directory for a deleted book crashed the sweep instead. A
133
+ guessed id now drives a row update only when metadata.db corroborates it
134
+ (the books row exists, the file sits in that book's own directory, and
135
+ when an EPUB row exists it carries the row's stored name); anything else
136
+ saves the repair in place with a warning and leaves the catalog
137
+ untouched. The resolver path needs no verification: its id comes from
138
+ the catalog's own path map.
139
+ - **The EPUB2-targeted fixes are gated on the package version.** Neither
140
+ `--strip-epub3-attrs` nor `--downgrade-epub3-tags` checked what the
141
+ package declared, so under `--all` every EPUB3 book in a sweep took the
142
+ EPUB2 cure: a repairable EPUB3 book gained a net-new error from a
143
+ stripped nav `epub:type` (whole repair rejected), and a legal
144
+ `epub:type="chapter"` was silently removed because the gate only sees
145
+ epubcheck counts. Both fixes now fire only on EPUB 2 packages (major
146
+ version 1 or 2) and are inert on EPUB 3 packages and when no version can
147
+ be read, where their target defects do not exist.
148
+ - **The anchored-attribute regex cluster closes with the gate** (the same
149
+ sweep's sharpest repair-pipeline finding): the attribute scrub runs
150
+ anchored on real start tags through the new
151
+ `transforms.strip_attrs_in_start_tags`, with CDATA sections and comments
152
+ protected, so prose mentioning `epub:type="chapter"` is no longer
153
+ deleted down to `Use here`; `strip_invalid_value` matches its attribute
154
+ name behind a `(?<![\w:.-])value` lookbehind, so `data-value="42"`
155
+ survives instead of becoming a malformed `<span data->`.
156
+
157
+ ### Repair-pipeline correctness (Phase 14, the 2026-09-08 sweep's backlog)
158
+
159
+ - **`--fix-id-colons` is consistent and honest.** Only the bare `id`
160
+ attribute is in scope (a lookbehind keeps `data-id` and `xml:id` values
161
+ untouched), the fragment of an external URL survives verbatim (it names a
162
+ position in another document), the NCX's `content src` fragments follow
163
+ the rename through the new `fix_ncx_src_fragments` so a ToC never dangles
164
+ against the ids it references, and only actual colons replaced are
165
+ counted (byte-identical reruns report 0 and no longer trigger the
166
+ re-encode the untouched-file guard exists to prevent).
167
+ - **The anchor pass runs last.** `strip_broken_anchors`' id snapshot used
168
+ to predate the unwrap fixes, so an id they deleted still counted as
169
+ present and fragments pointing at it survived dangling. The anchor pass
170
+ now runs after every content fix, against a snapshot that replicates
171
+ each id-moving fix ahead of it.
172
+ - **CDATA sections and comments are protected everywhere.**
173
+ `strip_broken_tags` and `unwrap_illegal_tags` joined the module's
174
+ protected-span machinery (the wrapper now forwards arguments, so the
175
+ protected_tags-carrying unwrap could be decorated); a leaked-looking tag
176
+ inside either is literal rendered text, not markup.
177
+ - **Non-UTF-8 documents are skipped verbatim.** A fix firing on a
178
+ windows-1252 or UTF-16 document used to decode with U+FFFD substitutes
179
+ and re-encode UTF-8 under a declaration still naming the old encoding:
180
+ usually well-formed, so the gate could not see the corruption. The write
181
+ loop strict-decodes first; failures are copied byte-for-byte, counted in
182
+ the new `non_utf8_docs_skipped` entry, and left for manual repair.
183
+ - **`--reserialize` requires an html root.** A broken non-HTML `.xml`
184
+ sidecar used to come back html/body-wrapped with `ns0:` prefixes,
185
+ structurally rewritten while staying well-formed; it is now returned
186
+ untouched.
187
+ - **Seven papercuts.** `fix_ncx_playorder` is anchored to `<navPoint>`
188
+ start tags (nav-label text is untouchable); mimetype fixes increment
189
+ `files_changed`; `fix_manifest_ids` and its reference rewrites are
190
+ quote-agnostic (a single-quoting OPF used to be a silent count-0 no-op;
191
+ the two shadowed `_IDREF_ATTR_RE` definitions collapsed into one);
192
+ the css selector boundary recognizes `svg|st`, `:is(st, w)`, and
193
+ `:where(...)` forms and the stylesheet scan collects `.xpgt` page
194
+ templates, so styled tags keep their unwrap protection; watermark
195
+ normalization decodes entities (`OceanofPDF.com&nbsp;` stamps now
196
+ match); the prolog and duplicate-xmlns fixes run under the
197
+ protected-span wrapper; and the archive comment field rides the rewrite.
198
+
199
+ Dry-run validation for this release: the 14 staged top500candidates plus
200
+ the 3 books_to_fix fixtures swept clean (9 accepted, 2 gate-refused
201
+ regressions, 0 errors), and a 200-candidate sample of the real library
202
+ (5,228 EPUBs) under `--all --workers 4` matched the documented verdict
203
+ mix (94 accepted, 84 equal, 16 gate-refused, 0 partial, 0 epubcheck
204
+ errors, 0 unreadable) with zero crashes. Read-only throughout; the
205
+ gate-refused books are the safety contract declining marginal repairs,
206
+ left untouched for manual work.
207
+
2
208
  ## v0.31.0 (2026-09-09)
3
209
 
4
210
  ### Cascade: install through cquarry 1.17's set_format
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "bindery-cli"
7
- version = "0.31.0"
7
+ version = "0.33.0"
8
8
  description = "Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library."
9
9
  readme = "README.md"
10
10
  classifiers = [