bindery-cli 0.29.0__tar.gz → 0.31.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/CLAUDE.md +1 -1
  2. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/PKG-INFO +3 -2
  3. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/README.md +1 -0
  4. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/patchnotes.md +29 -0
  5. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/pyproject.toml +2 -2
  6. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/roadmap.md +319 -4
  7. bindery_cli-0.31.0/scripts/FastSweepExtract.java +44 -0
  8. bindery_cli-0.31.0/scripts/sweep.sh +34 -0
  9. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/spec.md +5 -0
  10. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/__init__.py +1 -1
  11. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/cli.py +77 -4
  12. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/library.py +9 -8
  13. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_cli.py +122 -0
  14. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_library.py +5 -4
  15. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/uv.lock +5 -5
  16. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/.github/workflows/ci.yml +0 -0
  17. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/.github/workflows/publish.yml +0 -0
  18. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/.gitignore +0 -0
  19. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/AGENTS.md +0 -0
  20. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/LICENSE +0 -0
  21. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/docs/screenshots/library-sweep.png +0 -0
  22. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/logo.svg +0 -0
  23. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/run_tests.sh +0 -0
  24. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/FastSweep.java +0 -0
  25. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/fast_sweep.py +0 -0
  26. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_block_in_inline.py +0 -0
  27. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_css_illegal_tags.py +0 -0
  28. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_duplicate_playorder.py +0 -0
  29. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_empty_body.py +0 -0
  30. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_illegal_tags.py +0 -0
  31. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_invalid_ids.py +0 -0
  32. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_invalid_st_tag.py +0 -0
  33. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_invalid_value_attr.py +0 -0
  34. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_missing_images.py +0 -0
  35. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/scripts/find_missing_title.py +0 -0
  36. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/__main__.py +0 -0
  37. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/audit.py +0 -0
  38. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/epub.py +0 -0
  39. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/pagination.py +0 -0
  40. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/reserialize.py +0 -0
  41. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/transforms.py +0 -0
  42. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/validate.py +0 -0
  43. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/src/bindery/watermark.py +0 -0
  44. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/agda.epub +0 -0
  45. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/agda_fixed.epub +0 -0
  46. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/last_man_out.epub +0 -0
  47. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/last_man_out_fixed.epub +0 -0
  48. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/leaves_of_grass.epub +0 -0
  49. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/leaves_of_grass_fixed.epub +0 -0
  50. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/leaves_of_grass_st_fixed.epub +0 -0
  51. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/leaves_of_grass_tags_fixed.epub +0 -0
  52. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/muse_of_nightmares.epub +0 -0
  53. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/muse_of_nightmares_fixed.epub +0 -0
  54. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/poor_mans_fight.epub +0 -0
  55. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/poor_mans_fight_fixed.epub +0 -0
  56. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/test_block_in_inline.py +0 -0
  57. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/test_duplicate_playorder.py +0 -0
  58. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/test_empty_body.py +0 -0
  59. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/test_illegal_tags.py +0 -0
  60. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/test_invalid_ids.py +0 -0
  61. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/test_invalid_value.py +0 -0
  62. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/test_missing_title.py +0 -0
  63. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/unworthy_gods.epub +0 -0
  64. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/test_facility/unworthy_gods_fixed.epub +0 -0
  65. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/testing_facility/books_to_fix/.gitkeep +0 -0
  66. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/testing_facility/top500candidates/REPORT.md +0 -0
  67. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_audit.py +0 -0
  68. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_epub.py +0 -0
  69. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_pagination.py +0 -0
  70. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_reserialize.py +0 -0
  71. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_transforms.py +0 -0
  72. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_validate.py +0 -0
  73. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_version.py +0 -0
  74. {bindery_cli-0.29.0 → bindery_cli-0.31.0}/tests/test_watermark.py +0 -0
@@ -66,7 +66,7 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
66
66
  - `src/bindery/epub.py`: archive rewrite, NCX uid sync, RepairReport, mismatch detection, and the opt-in structural-repair plumbing (including the CSS precondition scan).
67
67
  - `src/bindery/validate.py`: epubcheck wrapper, the `gate` (improvement) and `no_worse` (no-regression, for the lossy strips) acceptance bars.
68
68
  - `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching) — the `(id)` directory-name regex is only the no-catalog fallback. Since v0.24.0 `install_format()` places the repaired file atomically and updates the `data` row through `cquarry.write.WritableCalibreDB` (remove+add in one `batch()`; `add_format` refuses duplicates); the external `calibredb` CLI is gone. Never reintroduce directory-name guessing as the primary source: renamed/mismatched directories would replace the wrong book.
69
- - `src/bindery/cli.py`: `repair` and `library` subcommands, including `--all` and `--install-to-calibre`; since v0.29.0 also the `run` verbs — `phase1` (pre-import vetting over loose files: the audit battery composed with one gated repair sweep; read-only until `--apply-lossy`, which IS the lossy-strip consent) and `phase3` (the scoped post-import apply step: `library --id --sweep --only all --apply --all --install-to-calibre` plus a pre/post summary; mechanically refuses unscoped sweeps with exit 2). Both take `--non-interactive` and surface open questions as `decisions_needed` in their JSON, never prompts. They drive the shipped `run_library` through the real parser, so no flag can drift between the verb and the subcommand it wraps.
69
+ - `src/bindery/cli.py`: `repair` and `library` subcommands, including `--all` and `--install-to-calibre`; since v0.29.0 also the `run` verbs: `phase1` (pre-import vetting over loose files: the audit battery composed with one gated repair sweep; read-only until `--apply-lossy`, which IS the lossy-strip consent) and `phase3` (the scoped post-import apply step: `library --id --sweep --only all --apply --all --install-to-calibre` plus a pre/post summary; mechanically refuses unscoped sweeps with exit 2). Both take `--non-interactive` and surface open questions as `decisions_needed` in their JSON, never prompts. They drive the shipped `run_library` through the real parser, so no flag can drift between the verb and the subcommand it wraps. `library --workers N` (v0.30.0) parallelizes only the sweep's candidate pass, in windows of N consumed in order so `--limit` stays lazy; the repair phase is never parallel (shared workdir, atomic-replacement contract).
70
70
  - `tests/`: transforms, end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic, archive integrity, spine classification), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, and the version pin.
71
71
 
72
72
  ## Conventions
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: bindery-cli
3
- Version: 0.29.0
3
+ Version: 0.31.0
4
4
  Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
5
5
  Author: vrnvctss
6
6
  License: MIT
@@ -13,7 +13,7 @@ Classifier: License :: OSI Approved :: MIT License
13
13
  Classifier: Programming Language :: Python :: 3.14
14
14
  Classifier: Topic :: Utilities
15
15
  Requires-Python: >=3.14
16
- Requires-Dist: cquarry>=1.9.0
16
+ Requires-Dist: cquarry>=1.17.0
17
17
  Requires-Dist: tqdm
18
18
  Requires-Dist: vir-tui>=2.2.0
19
19
  Provides-Extra: reserialize
@@ -196,6 +196,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
196
196
  - `--id <ids>` limits the sweep to a comma-separated list of Calibre book IDs, skipping the full library walk.
197
197
  - `--audit CSV` (the `fatals,errors,warnings,path` format produced by an epubcheck sweep) skips clean books so a run is fast. Paths are resolved on both sides, and a CSV that matches nothing triggers a loud warning instead of silently selecting zero books.
198
198
  - `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice. bindery-cli first tries a persistent Java daemon (`EpubcheckDaemon`) to eliminate the per-book JVM startup; in practice the daemon cannot boot on most installs (its classpath carries only epubcheck's launcher jar, not the `lib/` dependencies epubcheck itself puts on the classpath), and the run silently falls back to the standard per-book epubcheck subprocess at the usual seconds per book. Budget accordingly. Fixing the classpath is a recorded decision (it would also change the counts the gate measures), not a shipped behavior. Combine with `--only fatals` for a self-contained "find and fix the broken books" run.
199
+ - `--workers N` runs the sweep's candidate pass through N concurrent epubcheck workers (default 1: serial, unchanged). The subprocess releases the GIL, so threads parallelize the oracle honestly; books are checked in windows of N consumed in order, so the candidate set matches the serial sweep and `--limit` stays lazy within one window of overshoot. The repair phase stays serial: that is where the shared workdir and the atomic-replacement contract live.
199
200
  - `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
200
201
  - `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
201
202
  - `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
@@ -174,6 +174,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
174
174
  - `--id <ids>` limits the sweep to a comma-separated list of Calibre book IDs, skipping the full library walk.
175
175
  - `--audit CSV` (the `fatals,errors,warnings,path` format produced by an epubcheck sweep) skips clean books so a run is fast. Paths are resolved on both sides, and a CSV that matches nothing triggers a loud warning instead of silently selecting zero books.
176
176
  - `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice. bindery-cli first tries a persistent Java daemon (`EpubcheckDaemon`) to eliminate the per-book JVM startup; in practice the daemon cannot boot on most installs (its classpath carries only epubcheck's launcher jar, not the `lib/` dependencies epubcheck itself puts on the classpath), and the run silently falls back to the standard per-book epubcheck subprocess at the usual seconds per book. Budget accordingly. Fixing the classpath is a recorded decision (it would also change the counts the gate measures), not a shipped behavior. Combine with `--only fatals` for a self-contained "find and fix the broken books" run.
177
+ - `--workers N` runs the sweep's candidate pass through N concurrent epubcheck workers (default 1: serial, unchanged). The subprocess releases the GIL, so threads parallelize the oracle honestly; books are checked in windows of N consumed in order, so the candidate set matches the serial sweep and `--limit` stays lazy within one window of overshoot. The repair phase stays serial: that is where the shared workdir and the atomic-replacement contract live.
177
178
  - `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
178
179
  - `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
179
180
  - `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
@@ -1,4 +1,33 @@
1
1
  # bindery-cli Patch Notes
2
+ ## v0.31.0 (2026-09-09)
3
+
4
+ ### Cascade: install through cquarry 1.17's set_format
5
+
6
+ - **`--install-to-calibre` replaces the format row through
7
+ `WritableCalibreDB.set_format`** (cquarry 1.17's sanctioned remove+add
8
+ in one transaction) instead of composing `remove_format` + `add_format`
9
+ in a `batch()` by hand. Same contract as before: same `data.name`,
10
+ truthful size, OPF-resync queue, and a database failure still degrades
11
+ to the in-place save with a warning. The cquarry floor moves from
12
+ >=1.9.0 to **>=1.17.0** (`uv.lock` re-resolved), and the failure-mode
13
+ test now injects through `set_format`.
14
+
15
+ ## v0.30.0 (2026-09-06)
16
+
17
+ ### Phase 8 stretch: concurrent sweep workers
18
+
19
+ - **`library --workers N`**: the `--sweep` candidate pass runs through N
20
+ concurrent epubcheck workers (default 1: serial, byte-for-byte unchanged).
21
+ epubcheck runs in a subprocess that releases the GIL, so threads parallelize
22
+ the oracle honestly; this is the pass that measured ~4.4 s/book on the
23
+ 2026-08-27 full-library walk (5,070 EPUBs ≈ 6 hours serial). Books are
24
+ checked in windows of N consumed in input order, so the candidate set, the
25
+ before-measurements, and every emitted line match the serial sweep, and
26
+ `--limit` stays lazy within one window of overshoot. The repair phase stays
27
+ serial on purpose: that is where the shared workdir and the
28
+ atomic-replacement contract live. Without `--sweep`, `--workers` is a
29
+ no-op (with a note); counts below 1 are a usage error.
30
+
2
31
  ## v0.29.0 (2026-09-06)
3
32
 
4
33
  ### Phase 13: machine-readable audit and the acquisition run slices
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "bindery-cli"
7
- version = "0.29.0"
7
+ version = "0.31.0"
8
8
  description = "Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library."
9
9
  readme = "README.md"
10
10
  classifiers = [
@@ -27,7 +27,7 @@ authors = [
27
27
  dependencies = [
28
28
  "tqdm",
29
29
  "vir-tui>=2.2.0",
30
- "cquarry>=1.9.0",
30
+ "cquarry>=1.17.0",
31
31
  ]
32
32
 
33
33
  [project.optional-dependencies]
@@ -510,12 +510,19 @@ by hand.*
510
510
  release.
511
511
  - [x] **`audit --id` comma lists**: v0.19.0 took a single book id; support the
512
512
  same comma-separated form as the new `library --id` so the two modes match.
513
- - [ ] **Concurrent validation workers (stretch; not shipped)**: the 2026-08-27 full-library
513
+ - [x] **Concurrent validation workers (stretch; not shipped)**: the 2026-08-27 full-library
514
514
  walk measured ~4.4 s/book on a single daemon (5,070 EPUBs ≈ 6 h). Gate k
515
515
  validation workers behind `--workers N`, default 1 (behavior unchanged).
516
516
  *(Un-ticked in the 2026-09-02 docs re-read (NEW-AUDIT.md Stage 0): the box was
517
517
  ticked at the v0.23.0 ship, but no `--workers` code, test, or doc line ever
518
- landed. Still open as a stretch goal.)*
518
+ landed.)*
519
+ *(Shipped for real in v0.30.0: `library --workers N` runs the --sweep
520
+ candidate pass through N concurrent epubcheck workers, default 1 (serial,
521
+ unchanged). Threads parallelize the oracle honestly (the subprocess releases
522
+ the GIL); books are checked in windows of N consumed in input order, so the
523
+ candidate set matches the serial sweep and `--limit` stays lazy within one
524
+ window of overshoot. The repair phase stays serial, where the shared workdir
525
+ and the atomic-replacement contract live.)*
519
526
 
520
527
  Non-goals: no auto-trim of bloated ToCs (semantics-preserving charter); no
521
528
  content-doc synthesis; no PDF equivalent.
@@ -602,6 +609,58 @@ hand-rolled zipfile sweep.
602
609
  *(Fixed in v0.23.1; RETIRED entirely by Phase 11 in v0.24.0 — the calibredb
603
610
  subprocess is gone, so the crash class no longer exists.)*
604
611
 
612
+ ### watermark anchored pass can delete whole paragraphs of real prose (2026-09-08 sweep)
613
+ **Bug in `--strip-watermarks`:** the anchored stamp regex
614
+ `<a\b[^>]*?oceanofpdf[^>]*?>.*?</a>` is DOTALL with no tag budget, so an
615
+ unclosed stamp anchor (exactly the kind of broken book this tool repairs)
616
+ swallows everything up to the next unrelated `</a>`, and when no pure wrapper
617
+ exists the fallback deletes the entire match. Demonstrated: two paragraphs of
618
+ real content vanish with count 1; the output is well-formed with equal or
619
+ fewer fatals, so the `no_worse` bar accepts it. `watermark.py:207-213`.
620
+ **Fix:** require the anchored match to be tag-free (or stamp-length visible
621
+ text) before falling back to whole-match deletion; anything larger goes to
622
+ `decisions_needed` for manual repair. *(Unfixed as of the 2026-09-08 sweep.)*
623
+
624
+ ### roman-numeral detector reads ordinary English words as page numbers (2026-09-08 sweep)
625
+ **Bug in `--strip-pagination`:** `_ROMAN_RE = [ivxlcdm]{2,7}\Z` (IGNORECASE)
626
+ matches words like `mid`, `dim`, `mix`, `lid`, `civil` (`number_value("mid")
627
+ = 1499`), so a standalone `<p>mid</p>` followed by a lowercase-starting
628
+ paragraph is a confident page-number hit and is deleted and merged away.
629
+ epubcheck is blind to deleted text, so `no_worse` passes. `pagination.py:44`,
630
+ `70-78`, `180-184`.
631
+ **Fix:** require uppercase romans, or validate with a strict roman grammar
632
+ (`M?(D?C{0,3}|C[MD])(X?L{0,3}|L?X{0,3})(I?V{0,3}|V?I{0,3})`). *(Unfixed as of
633
+ the 2026-09-08 sweep.)*
634
+
635
+ ### install_format can write a wrong size into metadata.db for stray files (2026-09-08 sweep)
636
+ **Bug in `library --apply --install-to-calibre` without `--id`:**
637
+ `iter_epubs` yields uncatalogued stray `.epub` files inside book dirs; the
638
+ resolver returns None, the `(id)` directory-name guess fires, and the
639
+ remove+add batch re-registers the catalogued format name with the stray
640
+ file's size. Demonstrated on a synthetic library: the catalogued 999-byte
641
+ file untouched on disk while `data` recorded 8123 bytes. Silent,
642
+ order-dependent catalog corruption; a stale `(N)` directory for a deleted
643
+ book instead raises an uncaught `TypeError` at the `book["path"]` access.
644
+ `library.py:169-231`.
645
+ **Fix:** when the id came from the directory guess, verify the target
646
+ matches the DB's book path and format name before the batch; otherwise save
647
+ in place and warn without touching the row, and catch the `TypeError` path.
648
+ *(Unfixed as of the 2026-09-08 sweep.)*
649
+
650
+ ### EPUB2-targeted structural fixes fire on EPUB3 books under --all (2026-09-08 sweep)
651
+ **Bug in `--strip-epub3-attrs` / `--downgrade-epub3-tags` via the run verbs
652
+ and `--all`:** neither fix checks `package@version`, so every EPUB3 book in
653
+ the sweep gets `epub:type`/`aria-label`/`page-progression-direction` stripped
654
+ and EPUB3 tags downgraded. Two demonstrated outcomes: (a) a repairable EPUB3
655
+ book gains a net-new error from stripping `epub:type="toc"` off the nav and
656
+ the whole repair is rejected as a regression; (b) a legal count-neutral
657
+ `epub:type="chapter"` is silently removed because the gate only sees
658
+ epubcheck counts. phase1 and phase3 always pass `--all`.
659
+ `epub.py:251-260`, `903-937`, `957-967`; `cli.py:830`, `976-987`.
660
+ **Fix:** gate both fixes on the package version carried in `opf_text`, or
661
+ make them finding-driven (only strip what the before-pass epubcheck flagged).
662
+ *(Unfixed as of the 2026-09-08 sweep.)*
663
+
605
664
  ## Phase 11: Migrate install-to-calibre to native cquarry API (proposed 2026-08-31)
606
665
 
607
666
  *Context: `bindery library --install-to-calibre` currently shells out to the external `calibredb add_format` CLI binary. This is fragile (it crashed on 2026-08-31 due to a non-existent `--replace` flag) and bypasses the transaction and trigger-safety guarantees built into the `cquarry` library. Since bindery-cli already imports `cquarry` for path resolution, it should adopt the native write module.*
@@ -708,7 +767,7 @@ classes, no behavior change to existing modes.*
708
767
  Exit codes 0/1/2 per the `library` contract. No DRM scan, no duplicate
709
768
  screen, no manifest assembly (CalibreQuarry's by the frontend split).
710
769
  *(Shipped in v0.29.0. Two composed stages: the audit battery through its
711
- own --json payload, then ONE fused repair sweep with the --all set — the
770
+ own --json payload, then ONE fused repair sweep with the --all set: the
712
771
  skill ran watermark detection and repairability as two sweeps, the verb
713
772
  pays epubcheck once and reads watermark hits out of the per-book fix
714
773
  summary. Read-only until --apply-lossy; --backup passes through (keep
@@ -725,7 +784,7 @@ classes, no behavior change to existing modes.*
725
784
  the real parser, so its flags cannot drift from the subcommand it wraps.
726
785
  The pre/post summary sums before/after epubcheck counts over the swept
727
786
  books; books left partial or unreadable surface as decisions_needed.
728
- The verb is the apply step by design — the batch was vetted in phase 1,
787
+ The verb is the apply step by design: the batch was vetted in phase 1,
729
788
  and the gate plus the atomic-replacement contract still govern every
730
789
  write.)*
731
790
  - [x] **Non-interactive contract**: every `run` verb takes
@@ -750,3 +809,259 @@ Non-goals: no manifest format ownership (CalibreQuarry owns the
750
809
  `acquisition-manifest` schema and the orchestration; this repo emits reports
751
810
  and consumes id lists); no TUI beyond the existing audit rendering; no PDF/DJVU
752
811
  work (never this repo's charter).
812
+
813
+ ## Phase 14: hardening backlog from the 2026-09-08 audit sweep (proposed 2026-09-08, digging only)
814
+
815
+ *Context: a five-agent adversarial sweep of the whole repo (repair pipeline,
816
+ audit, CLI/library/validate, tests and scripts, docs). No code was changed;
817
+ every finding below was demonstrated against the shipped code, most with a
818
+ live run. The four sharpest bugs are written up under Bug Reports above; this
819
+ phase is the rest of the backlog. Nothing here is committed work: it is the
820
+ map of what the sweep found, for when work resumes.*
821
+
822
+ *Verification postscript (2026-09-08, an independent second batch re-derived
823
+ the five sharpest claims; all five CONFIRMED, with scoping notes): the
824
+ watermark loss is invisible to `no_worse` and can even clear a fatal, so the
825
+ book ships as a success; the window is the standalone `--strip-watermarks`
826
+ flag (the anchorless pass never fallback-deletes, and under `--all`
827
+ `--reserialize` closes the anchor first). The roman deletion needs a
828
+ 120+ char prose neighbor (`PROSE_MIN`) and a lowercase-starting continuation
829
+ paragraph, and no test acknowledges word-shaped romans. The `install_format`
830
+ exposure is exactly directory-mode `--apply --install-to-calibre`; phase3
831
+ passes `--id` and is not exposed. The `fix_id_colons` dangling NCX ref
832
+ survives the gate whenever colon fixes outnumber NCX references (the minimal
833
+ 1:1 case is refused as a noop), and the phantom count additionally triggers
834
+ the re-encode that the untouched-file guard exists to prevent. The EPUB3
835
+ hard reject needs a clean book; in already-error books the attribute strip
836
+ can be net-neutral and ships silently.*
837
+
838
+ ### Repair-pipeline correctness
839
+
840
+ - [ ] **Anchor attribute-name regexes to real start tags.** Three findings
841
+ share one root cause: bare attribute-name regexes that are not
842
+ quote-aware and not tag-anchored. `strip_invalid_value` matches the
843
+ `value` in `data-value="42"` (the `\b` matches between `-` and `v`)
844
+ and rewrites it to a malformed `<span data->`
845
+ (`transforms.py:375`); `_EPUB3_ATTR_RE` is not anchored to tags at all,
846
+ so `strip_epub3_attributes` deletes visible prose like
847
+ `Use epub:type="chapter" here` down to `Use here` (`epub.py:245-248`),
848
+ contradicting that fix's own docstring. Reuse the quote-aware
849
+ start-tag matcher + protected-span machinery the module already has,
850
+ and a `(?<![\w:.-])value` lookbehind.
851
+ - [ ] **Make `fix_id_colons` consistent and honest.** It rewrites fragments
852
+ of external URLs (`http://example.com/page#sec:1` becomes `#sec_1`,
853
+ breaking the link; `transforms.py:524`, `536`, docstring at 509 claims
854
+ otherwise) and rewrites colon-bearing `data-id`/`xml:id`; it never
855
+ translates the NCX's `content src` fragments, so renaming
856
+ manufactures dangling NCX refs end to end
857
+ (`epub.py:976-979` vs the NCX branch at `873-901`); and it counts
858
+ every matched `id` attribute even when no colon was replaced, so
859
+ byte-identical reruns report phantom changes (`transforms.py:513-516`).
860
+ - [ ] **Run anchor stripping last and protect CDATA/comments everywhere.**
861
+ `strip_broken_anchors`' id snapshot predates `unwrap_block_in_inline`
862
+ and `unwrap_illegal_tags`, which can delete ids the snapshot thinks
863
+ exist (`epub.py:980-997`); `strip_broken_tags` and
864
+ `unwrap_illegal_tags` are not wrapped in the module's own
865
+ protected-span machinery, so they edit inside CDATA (rendered text)
866
+ and comments (`transforms.py:444-450`, `619-623`).
867
+ - [ ] **Stop re-encoding non-UTF-8 documents.** Any fix that fires on a
868
+ windows-1252 or UTF-16 document decodes with `replace` and re-encodes
869
+ UTF-8, materializing mojibake under an XML declaration that still
870
+ names the old encoding; usually well-formed, so the gate cannot see
871
+ it (`epub.py:874`, `902`, `939`, `1027`). Detect and skip with a
872
+ report entry, per the project's manual-repair philosophy.
873
+ - [ ] **Scope `--reserialize` to HTML roots.** A broken non-HTML `.xml`
874
+ sidecar gets html5lib's HTML algorithm and comes back
875
+ html/body-wrapped with `ns0:` prefixes, structurally rewritten while
876
+ staying well-formed (`epub.py:938`, `reserialize.py:39-42`). Require
877
+ an `html` root (or manifest media-type) before rebuilding.
878
+ - [ ] **Smaller repair papercuts:** `fix_ncx_playorder` can rewrite
879
+ `playOrder="N"`-shaped text inside nav labels (unanchored pattern,
880
+ `transforms.py:326-328`); mimetype fixes land in `report.fixes`
881
+ without incrementing `files_changed` (`epub.py:837-840`);
882
+ `fix_manifest_ids` is a silent count-0 no-op on single-quoted OPFs
883
+ (`epub.py:59`); `css_protected_tags` misses namespaced
884
+ (`svg|st`) and functional (`:is(st, w)`) selectors, and doesn't scan
885
+ `.xpgt` templates (`transforms.py:419-421`, `epub.py:790-794`); the
886
+ watermark `_norm` doesn't unescape entities so `OceanofPDF.com&nbsp;`
887
+ stamps are missed; drop_duplicate_xmlns/strip_prolog_junk edit inside
888
+ comments; the archive comment field is dropped by the rewrite.
889
+
890
+ ### audit.py correctness
891
+
892
+ - [ ] **Read nav/NCX inside the open zip and resolve NCX srcs against the
893
+ NCX's own directory.** The post-close `_read` of a manifest-declared
894
+ but absent nav/NCX poisons `Book.corrupt`, so an otherwise healthy
895
+ book with a leftover `toc.ncx` manifest entry is branded CORRUPT
896
+ "re-source" (`audit.py:252-272`, `_read` at `173-177`). And `full()`
897
+ resolves NCX `content src` against the OPF's directory, not the NCX's,
898
+ so a spec-compliant nested NCX counts every target absent and can
899
+ reach the FRAGMENT "quarantine" alarm (`audit.py:187`, `205-213`,
900
+ `249`, `263-272`). Also `html.unescape` captured ToC hrefs, which are
901
+ currently compared raw against archive names (`audit.py:258`, `268-272`).
902
+ - [ ] **Make archive/spine first-class in library mode.** `run_library`'s
903
+ report loop iterates the content-analyzer tuple only, so the
904
+ `archive`/`spine` branches and their section builders are dead code:
905
+ a CRC-corrupt book prints `emptytext CLEAN` and exits 0 in library
906
+ mode while directory mode says CORRUPT and exits 1, contradicting the
907
+ v0.22.0 "its own verdict in every mode" promise (`audit.py:1565`,
908
+ `1783-1803`, `1520-1558`).
909
+ - [ ] **Harden the edges of the record pipeline:** a failed `--json` write
910
+ raises after the scan and loses the whole run's summary and exit code
911
+ (`audit.py:1616`, `2023-2043`); `--tag` tags books whose content hits
912
+ the run itself marked expected-foreign (`audit.py:1830-1832` vs the
913
+ rc filter); a decoded-href vs stored-name spine miss silently reads
914
+ as EMPTY on a book full of prose, with no diagnostic
915
+ (`audit.py:219`); DRM-encrypted entries report as CORRUPT
916
+ "re-source" instead of their own status (`audit.py:173-177`); the
917
+ broken-span heuristic treats duplicate trailing numbers
918
+ (`part1a/part1b/part2`) and duplicate spine itemrefs as FRAGMENT
919
+ (`audit.py:1066-1079`).
920
+ - [ ] **Delete the dead surfaces:** `audit.main()` + its argparse block
921
+ duplicate `cli.run_audit_cmd` (`audit.py:2212-2315`);
922
+ `analyze_brokentags` is unreachable (`audit.py:1260-1274`); duplicate
923
+ imports at the top of the file; and standalone `audit <dir>` silently
924
+ skips uppercase `.EPUB` files (`audit.py:1937`).
925
+ - [ ] **Smaller audit papercuts:** `_Blocks.handle_endtag` pops the stack
926
+ top regardless of which tag closed, so mis-nested blockquotes lose
927
+ their in_quote guard (`audit.py:739-743`); nav selection matches
928
+ `data-nav` by substring (`audit.py:196`); tagging failure can surface
929
+ an undocumented exit 3 (`audit.py:2203-2208`); the JSON `analyzers`
930
+ list omits the always-on archive/spine verdicts and the console
931
+ problem counter can disagree with `summary.problems`
932
+ (`audit.py:1603`); `--min-chars`/`--thin-chars` are not validated
933
+ against each other; duplicate zip entries silently resolve last-wins
934
+ with no note in the record.
935
+
936
+ ### Apply-path and oracle safety
937
+
938
+ - [ ] **Move the write-back inside the per-book exception isolation.**
939
+ `make_backup`/`atomic_replace`/`install_format` sit bare in the
940
+ sweep loop, so an ENOSPC or EACCES partway through a multi-hour run
941
+ aborts raw with no summary, no JSON, and no record of what was
942
+ already applied (`cli.py:498-508` vs `544-557`). Wrap the block,
943
+ record an error Outcome, and still emit the report; a run journal
944
+ (one line per applied book, resumable) is the stronger version.
945
+ - [ ] **Drive the exit code from partial books.** The documented contract
946
+ says exit 2 means "ran fine but some books are in trouble", but
947
+ partial books land in `still_fatal` and return 0, in both `library`
948
+ and phase3, while phase1 maps partial to problem/exit 2; the layers
949
+ disagree and scripts can miss trouble (`cli.py:260`, `526-534`, `634`,
950
+ `1030-1032`).
951
+ - [ ] **Give backups overwrite protection and keep them out of the
952
+ candidate set.** `make_backup` clobbers an existing backup, so a
953
+ second `--apply` destroys the only copy of the author original
954
+ (`library.py:37-41`); a `--backup` dir inside the library root gets
955
+ its `.epub`-named copies swept as candidates on the next run
956
+ (`library.py:29-34`). Refuse overwrite (or rotate) and reject an
957
+ in-tree backup path.
958
+ - [ ] **Make `--workers` parallelism real and bound the daemon.** The
959
+ epubcheck daemon holds its lock across the whole blocking
960
+ round-trip, so every worker serializes behind it and `--workers N`
961
+ degrades to serial exactly when epubcheck is present
962
+ (`validate.py:155-169`, `cli.py:213-255`); the daemon path ignores
963
+ `timeout` entirely, so a JVM hang hangs the sweep forever
964
+ (`validate.py:196`); a dead daemon never resets `_proc`, so every
965
+ later book reads as `error` for the life of the process
966
+ (`validate.py:160-170`); a failed daemon start leaks its
967
+ `/tmp/bindery-daemon-*` workdir and retries the javac compile per
968
+ book (`validate.py:130-153`); and `--only ncx` crashes with a raw
969
+ RuntimeError on encrypted archives because `_select` probes outside
970
+ the exception net (`epub.py:688`, `cli.py:184`).
971
+ - [ ] **Smaller apply papercuts:** `--limit -1` and a missing
972
+ `--audit` file raise tracebacks instead of usage errors
973
+ (`cli.py:434`, `373`); the fresh-format branch of `install_format`
974
+ registers the format under the name `repaired`
975
+ (`library.py:223-230`); phase1 ignores `--backup` without
976
+ `--apply-lossy` silently; the lossy override can upgrade a noop to
977
+ partial in the still-fatal listing (conservative direction only).
978
+
979
+ ### Tests
980
+
981
+ - [ ] **Add the missing direct tests for the safety contract:** `gate()` and
982
+ `no_worse()` have zero direct tests (the subtle fatal-fixing
983
+ error-unmasking branch is asserted nowhere); `_EpubcheckDaemon` is
984
+ completely untested (and hides the no-reset liveness bug above);
985
+ `--strip-broken-tags` has zero tests anywhere despite being one of
986
+ the three named lossy strips; `--strip-watermarks`' gated apply path
987
+ has no analogue of the pagination gate test; `fix_ncx_playorder`
988
+ (core, always-on) is untested; `atomic_replace`'s failure path
989
+ (temp cleaned, target untouched, re-raise) has no failure-injection
990
+ test; `--install-to-calibre` CLI wiring and library-mode `--tag`
991
+ end-to-end are untested.
992
+ - [ ] **Clean the weak 2%:** delete the three mid-file
993
+ `if __name__ == "__main__": unittest.main()` blocks
994
+ (`test_audit.py:806`, `:943`; `test_cli.py:475`) that make direct
995
+ file runs silently skip ~40 tests; drop the constant-assertion
996
+ `test_all_tuple_has_ocr`; consolidate the verbatim-duplicated
997
+ roman/number implementations and their twin test classes (audit copy
998
+ lacks pagination's year exclusion, which is its own question);
999
+ remove the dead `sys.argv` patches in three CLI tests; strengthen or
1000
+ delete `test_non_interactive_flag_wiring` (asserts argparse, not the
1001
+ runner); de-alias the `pagenum`/`emptytext`/`ocr` module aliases.
1002
+ - [ ] **Add a ruff step to run_tests.sh** so the local loop matches CI
1003
+ (`uvx ruff check . && uvx ruff format --check .`), and note the
1004
+ html5lib-dependent tests silently skip without uv.
1005
+
1006
+ ### Scripts and repo hygiene
1007
+
1008
+ - [ ] **Retire the subsumed find_* diagnostics.** All ten `scripts/find_*.py`
1009
+ detection wedges became shipped flags with tests
1010
+ (the css one says so in its own docstring port note); keeping both
1011
+ copies means future behavior changes land in one only. Delete (git
1012
+ preserves) or move to an attic note; also delete `sweep.sh` and
1013
+ `FastSweepExtract.java` (superseded by `fast_sweep.py --mode=extract`
1014
+ and `FastSweep.java`), keeping `fast_sweep.py` + `FastSweep.java`
1015
+ (a live, different tool: one JVM saturated across cores vs the
1016
+ library sweep's N subprocesses). The untracked `.class` files are
1017
+ local build leftovers; already gitignored.
1018
+ - [ ] **Decide what test_facility/ is.** It carries 10 tracked commercial
1019
+ EPUBs (muse-of-nightmares, last-man-out, etc.; only leaves-of-grass is
1020
+ public domain), a second never-run "suite" of bare functions that
1021
+ `unittest discover` reports as 0 tests, and it mutates the tracked
1022
+ books in place. `.gitignore`'s own comment says books are ignored to
1023
+ prevent IP leaks, but that rule only covers `testing_facility/`;
1024
+ `test_facility/` predates it. At minimum stop tracking the EPUBs and
1025
+ delete or properly convert the dead suite; stripping them from
1026
+ history is the thorough option (repo is public).
1027
+ - [ ] **Mention validate.py's Java epubcheck daemon in the docs.** It is
1028
+ the first oracle tried at runtime, it compiles `_DAEMON_JAVA`
1029
+ (`validate.py:73-105`) into a tempdir, and nothing in README/spec
1030
+ admits it exists; it is also the scariest untested code in the repo.
1031
+
1032
+ ### Documentation (drift is real but narrow)
1033
+
1034
+ - [ ] **Reconcile spec.md with the shipped opt-ins.** The contract's
1035
+ non-goals forbid exactly what `--reserialize` (v0.3.0, the lone
1036
+ html5lib dependency) and `--strip-bad-attrs` (v0.4.0) do, and neither
1037
+ has a spec section; the OPF half of `--fix-ids` is missing;
1038
+ "the OPF is left untouched" is true only of the default pass; the
1039
+ NCX sentence at spec.md:52-53 is garbled (a dangling "to the OPF
1040
+ unique identifier" tail). A contributor reading spec.md first would
1041
+ conclude two shipped flags violate the charter.
1042
+ - [ ] **Fix the exit-code contract.** README and spec document usage
1043
+ errors as exit 1, but argparse-level misuse exits 2 (the same code
1044
+ as "book in trouble"); either a custom parser exit or updated docs.
1045
+ - [ ] **README completeness:** document `--min-chars`, `--thin-chars`,
1046
+ `--max-doc-chars`, and `--limit` in the flag reference; fix the
1047
+ `--only fatals` bullet to "needs `--audit` or `--sweep`" (it
1048
+ currently contradicts the `--sweep` bullet one line over).
1049
+ - [ ] **CLAUDE.md's exception taxonomy** lists twelve structural repairs
1050
+ and three lossy strips but omits the four safe opt-ins
1051
+ (`--fix-ids`, `--add-img-alt`, `--strip-bad-attrs`,
1052
+ `--escape-unknown-entities`), which reads as a complete inventory.
1053
+
1054
+ ### Completeness verdict from the sweep
1055
+
1056
+ *The engine is close to complete for its charter: the five-fixture core is
1057
+ solid under adversarial attack (zip machinery, splice engine, entity
1058
+ handling, and the gate all held), the suite is fast, deterministic, and about
1059
+ 95% load-bearing, and the docs are truthful everywhere except spec.md's
1060
+ never-absorbed v0.3/v0.4 era. The gap between "almost as complete as it can
1061
+ be" and "done" is one coherent theme, not scattered work: several opt-in
1062
+ repairs trust their regexes where they should trust the parse (the
1063
+ anchored-tag cluster), the audit's newest machinery (archive/spine) is not
1064
+ wired into every mode it promises, and the apply path's failure modes (crash
1065
+ mid-run, backup clobber, daemon stall) are unpadded. Fix the four Bug
1066
+ Reports entries and the anchored-regex cluster before any further features;
1067
+ everything else is backlog, not danger.*
@@ -0,0 +1,44 @@
1
+ import java.io.File;
2
+ import java.io.PrintWriter;
3
+ import java.io.StringWriter;
4
+ import java.util.Scanner;
5
+ import java.util.ArrayList;
6
+ import java.util.List;
7
+ import com.adobe.epubcheck.api.EpubCheck;
8
+ import com.adobe.epubcheck.reporting.CheckingReport;
9
+
10
+ public class FastSweepExtract {
11
+ public static void main(String[] args) throws Exception {
12
+ Scanner scanner = new Scanner(System.in);
13
+ List<String> paths = new ArrayList<>();
14
+ while (scanner.hasNextLine()) {
15
+ String path = scanner.nextLine();
16
+ if (!path.trim().isEmpty()) paths.add(path);
17
+ }
18
+
19
+ paths.parallelStream().forEach(path -> {
20
+ File epub = new File(path);
21
+ if (!epub.exists()) return;
22
+ try {
23
+ StringWriter sw = new StringWriter();
24
+ PrintWriter out = new PrintWriter(sw);
25
+ CheckingReport report = new CheckingReport(out, epub.getName());
26
+ EpubCheck check = new EpubCheck(epub, report);
27
+ check.doValidate();
28
+ out.flush();
29
+ String text = sw.toString();
30
+ if (report.getErrorCount() > 0 || report.getFatalErrorCount() > 0 || report.getWarningCount() > 0) {
31
+ // Extract error codes (e.g. RSC-005, CSS-008)
32
+ java.util.regex.Matcher m = java.util.regex.Pattern.compile("([A-Z]{3}-\\d{3})").matcher(text);
33
+ List<String> codes = new ArrayList<>();
34
+ while (m.find()) {
35
+ codes.add(m.group(1));
36
+ }
37
+ synchronized(System.out) {
38
+ System.out.println(epub.getCanonicalPath() + " ||| " + String.join(",", codes));
39
+ }
40
+ }
41
+ } catch (Exception e) {}
42
+ });
43
+ }
44
+ }
@@ -0,0 +1,34 @@
1
+ #!/bin/bash
2
+ javac --release 25 -cp "/home/bdkl/.local/opt/epubcheck-5.3.0/epubcheck.jar:/home/bdkl/.local/opt/epubcheck-5.3.0/lib/*" FastSweepExtract.java
3
+ find "$HOME/docs/Calibre Library" -type f -name "*.epub" > all_epubs.txt
4
+ java -cp ".:/home/bdkl/.local/opt/epubcheck-5.3.0/epubcheck.jar:/home/bdkl/.local/opt/epubcheck-5.3.0/lib/*" FastSweepExtract < all_epubs.txt > raw_errors.txt
5
+ python3 -c '
6
+ import sys
7
+ from collections import Counter, defaultdict
8
+
9
+ error_counts = Counter()
10
+ file_errors = defaultdict(set)
11
+
12
+ with open("raw_errors.txt") as f:
13
+ for line in f:
14
+ line = line.strip()
15
+ if " ||| " not in line: continue
16
+ path, codes_str = line.split(" ||| ", 1)
17
+ if not codes_str: continue
18
+ codes = codes_str.split(",")
19
+ for c in codes:
20
+ error_counts[c] += 1
21
+ file_errors[c].add(path)
22
+
23
+ print("TOP ERRORS:")
24
+ top = error_counts.most_common(500)
25
+ for c, count in top:
26
+ print(f"{c}: {count}")
27
+
28
+ with open("top_errors.txt", "w") as out:
29
+ for c, count in top:
30
+ out.write(f"{c}: {count}\n")
31
+ # Save a few candidate files for each error
32
+ for f in list(file_errors[c])[:5]:
33
+ out.write(f" {f}\n")
34
+ ' > sweep_summary.txt
@@ -267,6 +267,11 @@ For a Calibre library (`Author/Title (id)/Title - Author.epub`):
267
267
  reported and counted as `unreadable`; it never aborts the rest of the sweep.
268
268
  - The `library` exit code is 0 for a clean sweep, 1 for a usage error, and 2 when any
269
269
  book was rejected, unreadable, or failed epubcheck, so scripts can detect trouble.
270
+ - With `--sweep`, `--workers N` runs the candidate-selection epubcheck pass through N
271
+ concurrent workers (default 1: serial, unchanged). Books are checked in windows of N
272
+ consumed in input order, so the candidate set and the before-measurements are identical
273
+ to the serial sweep; the repair phase is never parallel (the shared workdir and the
274
+ atomic-replacement contract live there).
270
275
 
271
276
  ### Native format installation (`--install-to-calibre`)
272
277
  Optionally, bindery-cli installs the repaired EPUB as the book's format through cquarry's write module (`WritableCalibreDB`): the file is placed atomically — an in-place replace over the catalogued file when one exists (same path, same `data.name`), or a fresh placement under the repaired file's stem otherwise — and the `data` row follows in one `batch()` transaction (`remove_format` + `add_format`, since `add_format` refuses duplicates), keeping the size truthful and queuing the book in `metadata_dirtied` so Calibre regenerates its sidecar .opf. The external `calibredb` CLI is no longer used (the v0.23.1 `--replace` crash class is gone with it). It automatically falls back to atomic filesystem replacement if a valid Calibre ID cannot be extracted, and a database failure degrades to the in-place save with a warning rather than losing the repair.
@@ -4,7 +4,7 @@ strip), epubcheck-gated.
4
4
  Single source of truth for the version. pyproject.toml must match VERSION below.
5
5
  """
6
6
 
7
- VERSION = "0.29.0"
7
+ VERSION = "0.31.0"
8
8
  __version__ = VERSION
9
9
 
10
10
  from .epub import RepairReport, repair_epub # noqa: E402
@@ -210,6 +210,51 @@ def _sweep_select(epubs, only: str, root: Path, checks: dict, *, quiet: bool):
210
210
  yield epub
211
211
 
212
212
 
213
+ def _sweep_select_parallel(
214
+ epubs, only: str, root: Path, checks: dict, *, quiet: bool, workers: int
215
+ ):
216
+ """--workers N: k concurrent epubcheck workers over the sweep pass.
217
+
218
+ epubcheck runs in a subprocess that releases the GIL, so threads
219
+ parallelize the oracle honestly (this pass measured ~4.4 s/book on the
220
+ 2026-08-27 full-library walk). Books are checked in windows of `workers`;
221
+ each window is consumed (and its keepers yielded) before the next one is
222
+ submitted, so a --limit still stops the sweep after at most one window of
223
+ overshoot instead of checking the whole tree. Within a window, results are
224
+ applied in input order, so the candidate set and every emitted line match
225
+ the serial sweep. The repair phase stays serial: that is where the shared
226
+ workdir and the atomic-replacement contract live."""
227
+ from concurrent.futures import ThreadPoolExecutor
228
+
229
+ todo = list(epubs)
230
+ if not todo:
231
+ return
232
+ with ThreadPoolExecutor(max_workers=workers) as ex:
233
+ done_count = 0
234
+ progress = (
235
+ None if quiet else tqdm(total=len(todo), desc="Sweeping", unit="book")
236
+ )
237
+ for start in range(0, len(todo), workers):
238
+ window = todo[start : start + workers]
239
+ futures = [ex.submit(run_epubcheck, epub) for epub in window]
240
+ for epub, fut in zip(window, futures, strict=True):
241
+ c = fut.result()
242
+ done_count += 1
243
+ if not quiet:
244
+ tqdm.write(f"[sweep] {epub.relative_to(root)}", file=sys.stderr)
245
+ progress.update(1)
246
+ if c is not None:
247
+ checks[epub] = c
248
+ if only == "fatals":
249
+ keep = c is None or c.fatals > 0
250
+ else: # only == "all": skip clean books
251
+ keep = c != CheckResult(0, 0, 0)
252
+ if keep:
253
+ yield epub
254
+ if progress is not None:
255
+ progress.close()
256
+
257
+
213
258
  # Everything a run did not (or could not) auto-repair; the --manual-list export.
214
259
  _MANUAL_STATUSES = frozenset(
215
260
  {"nochange", "equal", "partial", "reject", "error", "unreadable"}
@@ -353,11 +398,31 @@ def run_library(args) -> int:
353
398
  all_epubs = list(iter_epubs(root))
354
399
  audit_hits: list[Path] = []
355
400
  checks: dict[Path, CheckResult] = {}
356
- if args.sweep:
357
- iterator = (
358
- all_epubs if args.quiet else tqdm(all_epubs, desc="Sweeping", unit="book")
401
+ workers = getattr(args, "workers", 1)
402
+ if workers is None:
403
+ workers = 1
404
+ if workers < 1:
405
+ print(f"error: --workers must be >= 1, got {workers}", file=sys.stderr)
406
+ return 1
407
+ if not args.sweep and workers > 1:
408
+ print(
409
+ "note: --workers applies to the --sweep candidate pass; ignored here.",
410
+ file=sys.stderr,
359
411
  )
360
- selected = _sweep_select(iterator, args.only, root, checks, quiet=args.quiet)
412
+ if args.sweep:
413
+ if workers > 1:
414
+ selected = _sweep_select_parallel(
415
+ all_epubs, args.only, root, checks, quiet=args.quiet, workers=workers
416
+ )
417
+ else:
418
+ iterator = (
419
+ all_epubs
420
+ if args.quiet
421
+ else tqdm(all_epubs, desc="Sweeping", unit="book")
422
+ )
423
+ selected = _sweep_select(
424
+ iterator, args.only, root, checks, quiet=args.quiet
425
+ )
361
426
  else:
362
427
  selected = _select(all_epubs, args.only, audit, audit_hits)
363
428
  if args.limit is not None:
@@ -1309,6 +1374,14 @@ def build_parser() -> argparse.ArgumentParser:
1309
1374
  lib.add_argument(
1310
1375
  "--limit", type=int, help="process at most N candidates (for sampling)"
1311
1376
  )
1377
+ lib.add_argument(
1378
+ "--workers",
1379
+ type=int,
1380
+ default=1,
1381
+ metavar="N",
1382
+ help="concurrent epubcheck workers for the --sweep candidate pass "
1383
+ "(default 1: serial, unchanged). The repair phase stays serial",
1384
+ )
1312
1385
  lib.add_argument(
1313
1386
  "--quiet",
1314
1387
  action="store_true",
@@ -148,9 +148,9 @@ def install_format(
148
148
  With a book id, the repaired file is placed in the book's directory —
149
149
  an atomic replace over the catalogued file when one exists (same path,
150
150
  same ``data.name``; Calibre's layout never changes) — and the ``data``
151
- row follows through ``WritableCalibreDB``: ``remove_format`` +
152
- ``add_format`` in one ``batch()`` when the format exists (``add_format``
153
- refuses duplicates by design), a fresh ``add_format`` otherwise. The
151
+ row follows through ``WritableCalibreDB``: ``set_format`` (cquarry
152
+ 1.17's sanctioned remove+add in one transaction) when the format
153
+ exists, a fresh ``add_format`` otherwise. The
154
154
  row's size stays truthful and the book lands in ``metadata_dirtied``, so
155
155
  Calibre regenerates its sidecar .opf. Files are the caller's
156
156
  responsibility in cquarry; they are placed here, atomically, before the
@@ -210,11 +210,12 @@ def install_format(
210
210
  if row is not None:
211
211
  atomic_replace(target, new_file)
212
212
  placed = True
213
- with wdb.batch():
214
- wdb.remove_format(int(calibre_id), "EPUB")
215
- wdb.add_format(
216
- int(calibre_id), "EPUB", row["name"], new_file.stat().st_size
217
- )
213
+ # cquarry 1.17's set_format is the sanctioned remove+add;
214
+ # the explicit two-call batch this used to compose is
215
+ # retired with it.
216
+ wdb.set_format(
217
+ int(calibre_id), "EPUB", row["name"], new_file.stat().st_size
218
+ )
218
219
  else:
219
220
  book = wdb.conn.execute(
220
221
  "SELECT path FROM books WHERE id = ?", (int(calibre_id),)
@@ -1015,3 +1015,125 @@ class TestRunPhase3(unittest.TestCase):
1015
1015
  self.assertEqual(data["summary"]["repair"]["applied"], 1)
1016
1016
  self.assertEqual(data["decisions_needed"], [])
1017
1017
  self.assertIn("PHASE 3 SUMMARY", out.getvalue())
1018
+
1019
+
1020
+ class TestSweepWorkers(unittest.TestCase):
1021
+ """--workers N: concurrent epubcheck workers over the --sweep candidate
1022
+ pass (Phase 8 stretch). Default 1 is the serial sweep; N > 1 must select
1023
+ the same candidates in the same order, reuse the sweep result as the
1024
+ book's before-measurement, and keep --limit lazy within one window of
1025
+ overshoot."""
1026
+
1027
+ def test_flag_defaults_to_one(self):
1028
+ parser = build_parser()
1029
+ self.assertEqual(parser.parse_args(["library", "/tmp"]).workers, 1)
1030
+ self.assertEqual(
1031
+ parser.parse_args(["library", "/tmp", "--workers", "4"]).workers, 4
1032
+ )
1033
+
1034
+ def test_invalid_worker_count_is_a_usage_error(self):
1035
+ err = io.StringIO()
1036
+ with redirect_stdout(io.StringIO()), redirect_stderr(err):
1037
+ rc = main(["library", "/tmp", "--workers", "0", "--no-validate"])
1038
+ self.assertEqual(rc, 1)
1039
+ self.assertIn("--workers", err.getvalue())
1040
+
1041
+ def test_parallel_sweep_selects_like_the_serial_one(self):
1042
+ # a path-keyed oracle: threads complete in any order, but every
1043
+ # book's result is pinned to the book, so determinism survives
1044
+ def oracle(path):
1045
+ return (
1046
+ CheckResult(1, 0, 0) if path.name == "a.epub" else CheckResult(0, 0, 0)
1047
+ )
1048
+
1049
+ with tempfile.TemporaryDirectory() as td:
1050
+ for name in ("a", "b", "c"):
1051
+ build(Path(td) / f"{name}.epub")
1052
+ jout = Path(td) / "report.json"
1053
+ out, err = io.StringIO(), io.StringIO()
1054
+ with (
1055
+ mock.patch("bindery.cli.epubcheck_available", return_value=True),
1056
+ mock.patch("bindery.cli.run_epubcheck", side_effect=oracle),
1057
+ redirect_stdout(out),
1058
+ redirect_stderr(err),
1059
+ ):
1060
+ rc = main(
1061
+ [
1062
+ "library",
1063
+ td,
1064
+ "--sweep",
1065
+ "--only",
1066
+ "fatals",
1067
+ "--workers",
1068
+ "3",
1069
+ "--json",
1070
+ str(jout),
1071
+ ]
1072
+ )
1073
+ data = json.loads(jout.read_text())
1074
+ self.assertEqual(rc, 0)
1075
+ statuses = {Path(b["path"]).name: b["status"] for b in data["books"]}
1076
+ self.assertEqual(list(statuses), ["a.epub"])
1077
+ # the sweep result was reused as the before-measurement and the gate
1078
+ # accepted the no-op-fix candidate exactly as the serial sweep would
1079
+ self.assertEqual(statuses["a.epub"], "accept")
1080
+ self.assertEqual(data["summary"]["accepted"], 1)
1081
+
1082
+ def test_limit_stays_lazy_within_one_window(self):
1083
+ checked: list[str] = []
1084
+
1085
+ def oracle(path):
1086
+ checked.append(path.name)
1087
+ return CheckResult(1, 0, 0)
1088
+
1089
+ with tempfile.TemporaryDirectory() as td:
1090
+ for name in ("a", "b", "c", "d", "e"):
1091
+ build(Path(td) / f"{name}.epub")
1092
+ out, err = io.StringIO(), io.StringIO()
1093
+ with (
1094
+ mock.patch("bindery.cli.epubcheck_available", return_value=True),
1095
+ mock.patch("bindery.cli.run_epubcheck", side_effect=oracle),
1096
+ redirect_stdout(out),
1097
+ redirect_stderr(err),
1098
+ ):
1099
+ rc = main(
1100
+ [
1101
+ "library",
1102
+ td,
1103
+ "--sweep",
1104
+ "--only",
1105
+ "all",
1106
+ "--limit",
1107
+ "1",
1108
+ "--workers",
1109
+ "2",
1110
+ ]
1111
+ )
1112
+ self.assertEqual(rc, 0)
1113
+ # the first window (a, b) is checked; the keeper limit of 1 stops the
1114
+ # sweep there: at most one window of overshoot, never the whole tree.
1115
+ # (2 sweep calls + 1 after-validation on the candidate: the fixture's
1116
+ # &nbsp; always gives the core pass a fix, so the candidate validates.)
1117
+ self.assertEqual(len(checked), 3)
1118
+
1119
+ def test_serial_limit_checks_exactly_the_limit(self):
1120
+ checked: list[str] = []
1121
+
1122
+ def oracle(path):
1123
+ checked.append(path.name)
1124
+ return CheckResult(1, 0, 0)
1125
+
1126
+ with tempfile.TemporaryDirectory() as td:
1127
+ for name in ("a", "b", "c", "d", "e"):
1128
+ build(Path(td) / f"{name}.epub")
1129
+ out, err = io.StringIO(), io.StringIO()
1130
+ with (
1131
+ mock.patch("bindery.cli.epubcheck_available", return_value=True),
1132
+ mock.patch("bindery.cli.run_epubcheck", side_effect=oracle),
1133
+ redirect_stdout(out),
1134
+ redirect_stderr(err),
1135
+ ):
1136
+ rc = main(["library", td, "--sweep", "--only", "all", "--limit", "1"])
1137
+ self.assertEqual(rc, 0)
1138
+ # 1 sweep call + 1 after-validation on the single candidate
1139
+ self.assertEqual(len(checked), 2)
@@ -203,17 +203,18 @@ class TestInstallFormat(unittest.TestCase):
203
203
 
204
204
  def test_replace_survives_a_database_error(self):
205
205
  # A database failure must never lose the repair: the file is saved in
206
- # place and the warning says the row is stale. add_format fails after
207
- # the batch's remove_format, so the whole batch rolls back.
206
+ # place and the warning says the row is stale. set_format (cquarry
207
+ # 1.17's sanctioned remove+add) fails and rolls its own transaction
208
+ # back, so the old row survives untouched.
208
209
  import contextlib
209
210
  import io
210
211
 
211
- with mock.patch("cquarry.write.WritableCalibreDB.add_format") as m:
212
+ with mock.patch("cquarry.write.WritableCalibreDB.set_format") as m:
212
213
  m.side_effect = sqlite3.OperationalError("database is locked")
213
214
  with contextlib.redirect_stderr(io.StringIO()):
214
215
  install_format(self.epub, self.new, CalibreIdResolver(self.root))
215
216
  self.assertEqual(self.epub.read_bytes(), b"REPAIRED")
216
- # the batch rolled back: the old row survives untouched
217
+ # set_format rolled back: the old row survives untouched
217
218
  self.assertEqual(self._rows(), [("EPUB", "Title - Author", 10)])
218
219
 
219
220
  def test_guess_fallback_uses_calibre_dbpath(self):
@@ -4,7 +4,7 @@ requires-python = ">=3.14"
4
4
 
5
5
  [[package]]
6
6
  name = "bindery-cli"
7
- version = "0.29.0"
7
+ version = "0.31.0"
8
8
  source = { editable = "." }
9
9
  dependencies = [
10
10
  { name = "cquarry" },
@@ -19,7 +19,7 @@ reserialize = [
19
19
 
20
20
  [package.metadata]
21
21
  requires-dist = [
22
- { name = "cquarry", specifier = ">=1.9.0" },
22
+ { name = "cquarry", specifier = ">=1.17.0" },
23
23
  { name = "html5lib", marker = "extra == 'reserialize'", specifier = ">=1.1" },
24
24
  { name = "tqdm" },
25
25
  { name = "vir-tui", specifier = ">=2.2.0" },
@@ -37,11 +37,11 @@ wheels = [
37
37
 
38
38
  [[package]]
39
39
  name = "cquarry"
40
- version = "1.9.0"
40
+ version = "1.17.0"
41
41
  source = { registry = "https://pypi.org/simple" }
42
- sdist = { url = "https://files.pythonhosted.org/packages/37/50/ce20ce828d0836e229735c4297bb3761aa69c2d98949c7199a0f8dd1995b/cquarry-1.9.0.tar.gz", hash = "sha256:9d10644cdef161be466e4bdfa3a6c7a4a863ffd902fdc1148ca7a8dc68458871", size = 134894, upload-time = "2026-09-02T21:45:12.036Z" }
42
+ sdist = { url = "https://files.pythonhosted.org/packages/4d/6b/e2c80511bf62e3b008c52d47a05b29e43e2782614cf211aadc8c965a2be1/cquarry-1.17.0.tar.gz", hash = "sha256:07ec6c43ebc11f8921645a963373537ef4808985c88ad885de5dfc05ba08d737", size = 197288, upload-time = "2026-09-09T21:48:07.208Z" }
43
43
  wheels = [
44
- { url = "https://files.pythonhosted.org/packages/c9/8a/f10ca4563be8e16034374c3f01fc2c61386defeee10c7f3bee08f678a0fd/cquarry-1.9.0-py3-none-any.whl", hash = "sha256:ffc200eeab39a21a8952c3944ad3739e9c80f03bc8bdd6a3ecf8b2487d3c38a3", size = 53911, upload-time = "2026-09-02T21:45:10.649Z" },
44
+ { url = "https://files.pythonhosted.org/packages/4d/5f/90f6fe1c24a3ce8616cb377cbedee3dfb82c4eca84cbb1ec8b02a7034cb1/cquarry-1.17.0-py3-none-any.whl", hash = "sha256:f01c5b4ecb0863c78379e9fc899fd0da7d2518e815ed2a9000acec09cbea054b", size = 70998, upload-time = "2026-09-09T21:48:05.798Z" },
45
45
  ]
46
46
 
47
47
  [[package]]
File without changes
File without changes
File without changes
File without changes
File without changes