bindery-cli 0.59.0__tar.gz → 0.61.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/AGENTS.md +5 -4
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/PKG-INFO +2 -2
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/README.md +1 -1
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/patchnotes.md +99 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/pyproject.toml +1 -1
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/roadmap.md +1 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/spec.md +6 -2
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/__init__.py +1 -1
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/cli.py +8 -4
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/epub.py +4 -1
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/transforms.py +14 -2
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/validate.py +33 -7
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_epub.py +22 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_transforms.py +47 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_validate.py +64 -6
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/uv.lock +1 -1
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/.github/FUNDING.yml +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/.github/workflows/ci.yml +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/.github/workflows/publish.yml +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/.gitignore +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/LICENSE +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/OBSERVE-RESEARCH.md +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/docs/screenshots/library-sweep.png +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/logo.svg +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/plugin/__init__.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/project.done +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/run_tests.sh +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/scripts/FastSweep.java +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/scripts/build_plugin.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/scripts/fast_sweep.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/__main__.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/audit.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/library.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/observe.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/pagination.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/reserialize.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/src/bindery/watermark.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_audit.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_cli.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_comment_hyphens.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_contamination.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_cover_tocdrift.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_flags_real_core.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_flags_wiring.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_help.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_library.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_matcher_hardening.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_observe.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_pagination.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_plugin.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_reserialize.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_space_renames.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_spine_guard.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_version.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_watermark.py +0 -0
- {bindery_cli-0.59.0 → bindery_cli-0.61.0}/tests/test_xml_hardening.py +0 -0
|
@@ -191,8 +191,9 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
|
|
|
191
191
|
(their gain is invisible to epubcheck, so it only forbids a regression, never demands
|
|
192
192
|
a measured improvement). The structural opt-ins go through the normal `gate` (their
|
|
193
193
|
gain IS visible, so a run with no measurable improvement is a noop and nothing is
|
|
194
|
-
applied) except the
|
|
195
|
-
`--fix-cover
|
|
194
|
+
applied) except the three whose gains sit on axes epubcheck does not measure:
|
|
195
|
+
`--fix-cover`, `--encode-url-spaces`, and `--fix-media-types` (the font half,
|
|
196
|
+
v0.56.0), accepted under the same `no_worse` bar with
|
|
196
197
|
the partial rule intact. One masked-axis rule (v0.51.0, issue #9, for the
|
|
197
198
|
NCX sync; generalized v0.57.0, issue #22, to every fired fix): when the gate
|
|
198
199
|
answers noop on a book that started AND stayed fatal, `process_book`
|
|
@@ -230,13 +231,13 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
|
|
|
230
231
|
character references before the fragment split (attribute text arrives
|
|
231
232
|
escaped; entries are raw; decoding first keeps the `#` inside `&`
|
|
232
233
|
from cutting the target), `_ITEM_TAG_RE`/`_SPINE_ITEM_RE`/`_GUIDE_REF_RE`/prune_dangling_edges' matcher accept an optional ns prefix (`<opf:item>`), and the renamed-reference spellings percent-encode `[`/`]` (`_encode_ref_path`). Since v0.52.0 (issue #10): `spine_dangles(src)` walks itemref -> manifest item -> archive entry through `_resolve_href`+`_norm_path` (None on an unreadable package: the veto needs evidence, never fires on a probe failure), and `prune_missing_manifest_items` derives its spine protection from the text it prunes (no snapshot parameter: fix_manifest_ids renames ids ahead of it, and the existence set it checks against must be the layout its hrefs are written in — `present_src` when space renames are active, matching the content-doc prune). Since v0.46.0 the opt-in selection is the `RepairFlags` dataclass (one field per flag, all off by default; the all-off instance IS the default pass): `repair_epub(src, dst, flags)` and `process_book(..., flags)` take one object instead of a ~25-kwarg signature, the EPUB2-targeted fixes' version gate swaps in a `dataclasses.replace` copy (never mutating the caller's instance), and the plugin's bare `repair_epub(src, dst)` call constructs exactly the all-off default pass. Since v0.55.0 (issue #16) the RSC-008 half lives beside the prune: `collect_undeclared_resources` (one whole-book census: content img/link, inline `<style>` url(), stylesheet url(), OPF guide refs) + `declare_missing_manifest_items` (sorted `bindery-dN` items before `</manifest>`, `_DECLARE_MEDIA_TYPES` blessed extensions only), gated and threaded through `RepairFlags.declare_missing`. Since v0.38.0/v0.39.0 (Phase 16): `generate_container` + the `--fix-container` gateway (insert after mimetype when missing, replace in-loop when stale; constant-epoch timestamp keeps bytes deterministic), `prune_dangling_edges` (prune_missing_manifest_items returns the pruned ids and the edges that pointed at them (spine@toc, media-overlay, fallback, EPUB2 cover meta) are rewritten so pruning cannot manufacture the regression the gate rejects), and `fix_manifest_media_types` (extension map + magic-byte confirmation via a `peek` callable over the open source zip; jpg/png/gif only), and `fix_cover_meta` (the ruling's deterministic half: dangling EPUB2 cover meta re-pointed from the guide's cover reference when exactly one manifest item carries that file, removed otherwise; accepted under no_worse like the lossy strips because cover wiring is invisible to epubcheck).
|
|
233
|
-
- `src/bindery/validate.py`: epubcheck wrapper, the `gate` (improvement) and `no_worse` (no-regression, for the lossy strips) acceptance bars. Since v0.55.0 (issue #15) the default oracle budget is size-scaled (`oracle_timeout_for`: 300s at or under 10 MB, +60s per started 10 MB above, 1800s cap), one subprocess retry follows a timed-out attempt (the daemon path's subprocess fallback is that attempt), and only a double timeout raises `OracleTimeout`, which the gate surfaces as `(epubcheck timeout)` instead of `(epubcheck failed)` (the sweep selectors catch it and keep the book a candidate).
|
|
234
|
+
- `src/bindery/validate.py`: epubcheck wrapper, the `gate` (improvement) and `no_worse` (no-regression, for the lossy strips) acceptance bars. Since v0.55.0 (issue #15) the default oracle budget is size-scaled (`oracle_timeout_for`: 300s at or under 10 MB, +60s per started 10 MB above, 1800s cap), one subprocess retry follows a timed-out attempt (the daemon path's subprocess fallback is that attempt), and only a double timeout raises `OracleTimeout`, which the gate surfaces as `(epubcheck timeout)` instead of `(epubcheck failed)` (the sweep selectors catch it and keep the book a candidate). Since v0.60.0 (issue #24) the size term counts decimal MB (10,440,245 bytes is 9.96 binary MiB, so the book the docs call 10.4 MB computed the flat 300s base with no size chunk), and the budget also scales on archive entry count (+60s per started 500 entries above 1,000; a path that does not open as a zip falls back to size-only): the Summa Theologica is entry-count-bound, not bytes-bound (3,831 entries / 3,808 spine docs; measured 384s source and 445s candidate), and now computes 720s.
|
|
234
235
|
- `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching); the `(id)` directory-name regex is only the no-catalog fallback. Since v0.24.0 `install_format()` places the repaired file atomically and updates the `data` row through `cquarry.write.WritableCalibreDB` (since v0.31.0 via `set_format`, cquarry 1.17's sanctioned remove+add in one transaction); the external `calibredb` CLI is gone. Since v0.32.0 a guessed id drives a row update only when metadata.db corroborates it (`_verify_guess`: books row exists, file in the book's own directory, catalogued `data.name` when a row exists); anything else saves in place and leaves the catalog untouched: never reintroduce directory-name guessing as the primary source: renamed/mismatched directories would replace the wrong book. Since 2026-09-15 `make_backup` takes the opt-in `--backup-keep N` ring: at most N backup files per book, the author original `.bak` never deleted, newest content at the highest name; default remains an unbounded rotation.
|
|
235
236
|
- `src/bindery/cli.py`: `repair` and `library` subcommands, including `--all` and `--install-to-calibre`; since v0.52.0 `process_book` carries the spine-resolution veto (issue #10) above every verdict path, promotions included: a candidate whose spine dangles more than the source's is `reject` with `(spine collapse)` appended to the summary, whatever the counts say. `repair --json` emits the library per-book record vocabulary (2026-09-15; since v0.45.0 every per-book record, the phase1 payload included, also carries the structured fix breakdown: the `fixes` dict plus `ncx_uid_synced`/`watermark_refusals` off the `Outcome`, and `_phase1_decisions` reads that data instead of substring-matching the rendered `summary`, which stays for human eyes; CalibreQuarry's lossy-consent mirror is the waiting consumer, floor `bindery>=0.45.0`); since v0.29.0 also the `run` verbs: `phase1` (pre-import vetting over loose files: the audit battery composed with one gated repair sweep; read-only until `--apply-lossy`, which IS the lossy-strip consent) and `phase3` (the scoped post-import apply step: `library --id --sweep --only all --apply --all --install-to-calibre` plus a pre/post summary; mechanically refuses unscoped sweeps with exit 2; since 2026-09-15 the summary sums applied/equal states only and carries rejected projections as `rejected_projection`). Plus `bindery doctor` (2026-09-15): the environment self-check that imports none of the VirInvictus stack, never raises, and always exits 0. The `observe` verb (v0.53.0) writes schema v2 artifacts since v0.55.0 (issue #13): `codes.messages` (renamed from the mislabeled `occurrences`) counts aggregated epubcheck JSON message entries, `codes.occurrences` now carries the true per-site scale (summed `locations`), a `counting` block states both scales in the artifact, and `--against` accepts v1 or v2 (the trend keys exist in both). Both run verbs take `--non-interactive` and surface open questions as `decisions_needed` in their JSON, never prompts. They drive the shipped `run_library` through the real parser, so no flag can drift between the verb and the subcommand it wraps. Since v0.46.0 `cli._flags_from_args(args)` is the one place the argparse names map onto `RepairFlags` fields (each `or --all`), shared by `run_library` and `run_repair`; the old per-verb ~25-kwarg blocks are gone. Since v0.47.1 the help is two-level (the --help-repairs reference is
|
|
236
237
|
custom-rendered and colors itself with the CPython 3.14 argparse theme,
|
|
237
238
|
gated by argparse's own _colorize detection so the levels always agree) and the repair-flag inventory is one table: `_REPAIR_FLAG_TABLE` (dest + one-line short help + long reference text) registers the compact grouped `--help` (argparse argument groups, hand-short usage line) and `--help-repairs` renders the long descriptions (a custom argparse Action, deliberately NOT store_true so the wiring test's selection count ignores it); adding a flag means adding a table row. Since v0.48.0 the metadata.db doors are enforced: `_require_closed_calibre` (pgrep fail-closed, exit 1) guards `audit --tag` and `library --apply --install-to-calibre` (run phase3 transitively), and `--help-json` (a custom argparse Action like --help-repairs, never colored) dumps the per-verb exit contract and the door list; since v0.48.1 the per-verb exit contracts tell the code's truth (repair: 0 probed/written incl. partial / 1 trouble, never 2; audit: 0/1/2 plus 3 for findings+write-failure; library and the run slices: the unified 0/2/1); since v0.49.0 the single-file verb's no-output form is a probe (the gate runs, the record's `output` is null and `applied` false, nothing is written anywhere; the old implicit `(repaired).epub` sibling was issue #7's import litter in a vetted tree, and writing now needs an explicit DST). `library --workers N` (v0.30.0) parallelizes only the sweep's candidate pass, in windows of N consumed in order so `--limit` stays lazy; the repair phase is never parallel (shared workdir, atomic-replacement contract). Since v0.40.0 `main()` guards the VirInvictus stack: a `ModuleNotFoundError` naming `vir_tui`/`cquarry` (marker-gated to 3.14+ on the 3.12 floor) becomes one stderr line and trouble exit 2 instead of a traceback; anything else missing still raises.
|
|
238
239
|
- `plugin/__init__.py` + `scripts/build_plugin.py`: the Bindery Repair Calibre plugin (v0.37.0; identity `Bindery Repair` / `bindery_repair`, approved and built per the scoped spec in roadmap.md Phase 3). The builder vendors `transforms.py`, `epub.py`, `pagination.py`, `watermark.py`, `reserialize.py` byte-identically into the zip (the zip root is a package, so their relative imports resolve unchanged; the suite's drift test pins the equality) with the version tuple substituted from the single-source VERSION; `publish.yml` attaches `BinderyRepair-v<VERSION>.zip` to each release. The plugin runs ONLY the default pass (epubcheck cannot gate inside Calibre), never raises, is byte-idempotent (zero fixes returns the original path), testzip()-verifies its output, logs one line per book, and takes JSON config via `site_customization` (`log`, `log_path`, `max_size_mb` default 150, `max_log_mb` rotation cap default 2, `epubcheck_path` experimental validation default OFF). Load it in tests through the `tests/test_plugin.py` stubs: the calibre namespace is installed in sys.modules and the package loads with a two-dir `__path__` (plugin dir first, then src/bindery) exactly like calibre's zipplugin loader shape. **Plugin/interpreter compatibility (v0.40.0):** v0.39.0 shipped PEP 758 unparenthesized except tuples, and the plugin died with a bare SyntaxError on every released Calibre (7.x and the whole 8.x series embed Python 3.11, 9.0+ embeds 3.14; from calibre's bypy sources.json, checked against the upstream clone). The core is now grammar-pinned: parenthesized except tuples everywhere, `ruff`'s `target-version = "py312"` pin (inference from requires-python would rewrite the parens back off), `tests/test_version.py`'s parse guard, and CI's `plugin-compat` job byte-compiling the built zip under 3.11/3.12/3.13/3.14. `minimum_calibre_version` is (7, 0, 0): the oldest series whose interpreter CI covers, replacing the hollow (2, 0, 0); `supported_platforms` stays `["linux"]` (developed and tested on Linux only). Any new syntax in the vendored modules must clear the guard before it ships.
|
|
239
|
-
- `tests/`: transforms (including the shared duplicate-id rename), end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic/completeness, archive integrity, spine classification, contamination since v0.50.0), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, the version pin and the syntax-floor grammar guard, and the Calibre plugin (build drift + run behavior under calibre stubs). Since the v0.46.0 follow-ups: `tests/test_flags_wiring.py` reflectively pins the RepairFlags inventory to the argparse dests and `_flags_from_args` mapping (a flag added without a field, a field without a flag, or a remapped line fails there), the all-off defaults, and the version gate's caller-immutability contract (test_epub's `TestVersionGateKeepsFlagsIntact`); `tests/test_comment_hyphens.py` carries the RSC-016 fix; `tests/test_flags_real_core.py` drives every repair flag from argv through the unmocked core (--no-validate; the gate itself stays pinned by the mocked-oracle tests and runs for real wherever epubcheck exists). `tests/test_contamination.py` (v0.50.0) carries the three contamination sweeps, the advisory contract, the `_dir_dispatch` key pin, and the cover-routing regression. The v0.46.0 trio (test_flags_wiring, test_comment_hyphens, test_flags_real_core) rides CI's core-compat 3.12/3.13 leg; the hand-listed module list there must gain an entry for every new stack-free test module (audit-side test modules like test_contamination stay on the 3.14 leg by convention). The v0.55.0 batch (issues #11-#16): `tests/test_validate.py` pins the size-scaled timeout and the retry-then-OracleTimeout contract, `tests/test_observe.py` the v2 line format (`CODE:messages:occurrences`), the renamed codes block, the counting envelope, and v1 `--against` acceptance, `tests/test_epub.py::TestDeclareMissingItems` the RSC-008 declarer (sorted deterministic items, blessed media types only, id collisions prefixed, schemes/fragments/spaces/declared skipped), `tests/test_transforms.py::TestFixDuplicateIds` the content-doc rename, and `tests/test_flags_real_core.py` drives `--fix-duplicate-ids`, `--declare-missing-items`, and the extended `--strip-epub3-attrs` set through the real argv-to-fix path. The v0.59.0 addition (issue #23): `tests/test_epub.py::TestPruneDanglingNcxTargets` pins the node-exact prune rule (dead leaf dropped, dead parent spliced above live children, dead subtree one span, scheme/bare-fragment/contentless/comment-hidden untouched, escaped-ampersand src resolves, pageTarget and legacy navTarget covered, wrapper-crossing and unclosed walks abort, a wholesale-dead container refuses) plus the end-to-end prune+resequence, and `tests/test_flags_real_core.py` drives `--prune-dangling-ncx-targets` through argv.
|
|
240
|
+
- `tests/`: transforms (including the shared duplicate-id rename), end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic/completeness, archive integrity, spine classification, contamination since v0.50.0), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, the version pin and the syntax-floor grammar guard, and the Calibre plugin (build drift + run behavior under calibre stubs). Since the v0.46.0 follow-ups: `tests/test_flags_wiring.py` reflectively pins the RepairFlags inventory to the argparse dests and `_flags_from_args` mapping (a flag added without a field, a field without a flag, or a remapped line fails there), the all-off defaults, and the version gate's caller-immutability contract (test_epub's `TestVersionGateKeepsFlagsIntact`); `tests/test_comment_hyphens.py` carries the RSC-016 fix; `tests/test_flags_real_core.py` drives every repair flag from argv through the unmocked core (--no-validate; the gate itself stays pinned by the mocked-oracle tests and runs for real wherever epubcheck exists). `tests/test_contamination.py` (v0.50.0) carries the three contamination sweeps, the advisory contract, the `_dir_dispatch` key pin, and the cover-routing regression. The v0.46.0 trio (test_flags_wiring, test_comment_hyphens, test_flags_real_core) rides CI's core-compat 3.12/3.13 leg; the hand-listed module list there must gain an entry for every new stack-free test module (audit-side test modules like test_contamination stay on the 3.14 leg by convention). The v0.55.0 batch (issues #11-#16): `tests/test_validate.py` pins the size-scaled timeout and the retry-then-OracleTimeout contract, `tests/test_observe.py` the v2 line format (`CODE:messages:occurrences`), the renamed codes block, the counting envelope, and v1 `--against` acceptance, `tests/test_epub.py::TestDeclareMissingItems` the RSC-008 declarer (sorted deterministic items, blessed media types only, id collisions prefixed, schemes/fragments/spaces/declared skipped), `tests/test_transforms.py::TestFixDuplicateIds` the content-doc rename, and `tests/test_flags_real_core.py` drives `--fix-duplicate-ids`, `--declare-missing-items`, and the extended `--strip-epub3-attrs` set through the real argv-to-fix path. The v0.59.0 addition (issue #23): `tests/test_epub.py::TestPruneDanglingNcxTargets` pins the node-exact prune rule (dead leaf dropped, dead parent spliced above live children, dead subtree one span, scheme/bare-fragment/contentless/comment-hidden untouched, escaped-ampersand src resolves, pageTarget and legacy navTarget covered, wrapper-crossing and unclosed walks abort, a wholesale-dead container refuses) plus the end-to-end prune+resequence, and `tests/test_flags_real_core.py` drives `--prune-dangling-ncx-targets` through argv. The v0.60.0 addition (issue #24): `tests/test_validate.py` pins the decimal-MB size term (exactly 10 MiB earns its 360s chunk) and the entry term (the 3,831-entry shape at <= 10 MB computes 660s; the full Summa shape computes 720s; exactly 1,000 entries stays at the size budget; 1,001 adds the first chunk; 2,500 computes 540s).
|
|
240
241
|
|
|
241
242
|
## Conventions
|
|
242
243
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bindery-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.61.0
|
|
4
4
|
Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
|
|
5
5
|
Project-URL: Homepage, https://github.com/VirInvictus/bindery-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/VirInvictus/bindery-cli
|
|
@@ -57,7 +57,7 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
|
|
|
57
57
|
- **`--strip-bad-attrs`**: drop attributes that are invalid XML (a name starting with a digit, or a namespaced name whose prefix is never declared, like Office VML `v:shapes`). Surgical and a no-op on well-formed files.
|
|
58
58
|
- **`--escape-unknown-entities`**: escape entity names that are not in the HTML5 table (`&foo;` becomes `&foo;`), which renders exactly as browsers already render an unknown entity. Documents whose DOCTYPE carries an internal subset are skipped wholesale, since a subset can declare custom entities.
|
|
59
59
|
- **`--fix-id-colons`**: translates illegal colons in `id="X:Y"` and their matching internal `#X:Y` fragment references to valid underscores. External URL fragments are left verbatim (they name positions in other documents), `data-id` values are never touched, and the NCX's `content src` fragments follow the rename so the table of contents never dangles against the ids it references.
|
|
60
|
-
- **`--fix-empty-body`**:
|
|
60
|
+
- **`--fix-empty-body`**: gives a strictly empty `<body></body>` one minimal paragraph (`<p> </p>`) to clear the "body incomplete" error. The numeric character reference needs no DTD and the wrapping `p` is the block element XHTML 1.1's body content model requires.
|
|
61
61
|
- **`--fix-missing-title`**: injects a `<title>Unknown</title>` fallback in the `<head>` if missing, handling both empty `<title/>` self-closing tags and entirely absent tags.
|
|
62
62
|
- **`--unwrap-block-in-inline`**: safely unwraps `<span>` tags that illegally contain a block-level element (e.g. `<div>` or `<p>`), leaving the block element intact.
|
|
63
63
|
- **`--strip-invalid-value`**: systematically strips invalid `value="..."` attributes from elements like `<div>`, `<span>`, `<p>`, etc.
|
|
@@ -30,7 +30,7 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
|
|
|
30
30
|
- **`--strip-bad-attrs`**: drop attributes that are invalid XML (a name starting with a digit, or a namespaced name whose prefix is never declared, like Office VML `v:shapes`). Surgical and a no-op on well-formed files.
|
|
31
31
|
- **`--escape-unknown-entities`**: escape entity names that are not in the HTML5 table (`&foo;` becomes `&foo;`), which renders exactly as browsers already render an unknown entity. Documents whose DOCTYPE carries an internal subset are skipped wholesale, since a subset can declare custom entities.
|
|
32
32
|
- **`--fix-id-colons`**: translates illegal colons in `id="X:Y"` and their matching internal `#X:Y` fragment references to valid underscores. External URL fragments are left verbatim (they name positions in other documents), `data-id` values are never touched, and the NCX's `content src` fragments follow the rename so the table of contents never dangles against the ids it references.
|
|
33
|
-
- **`--fix-empty-body`**:
|
|
33
|
+
- **`--fix-empty-body`**: gives a strictly empty `<body></body>` one minimal paragraph (`<p> </p>`) to clear the "body incomplete" error. The numeric character reference needs no DTD and the wrapping `p` is the block element XHTML 1.1's body content model requires.
|
|
34
34
|
- **`--fix-missing-title`**: injects a `<title>Unknown</title>` fallback in the `<head>` if missing, handling both empty `<title/>` self-closing tags and entirely absent tags.
|
|
35
35
|
- **`--unwrap-block-in-inline`**: safely unwraps `<span>` tags that illegally contain a block-level element (e.g. `<div>` or `<p>`), leaving the block element intact.
|
|
36
36
|
- **`--strip-invalid-value`**: systematically strips invalid `value="..."` attributes from elements like `<div>`, `<span>`, `<p>`, etc.
|
|
@@ -1,5 +1,104 @@
|
|
|
1
1
|
# bindery-cli Patch Notes
|
|
2
2
|
|
|
3
|
+
## v0.61.0 (2026-11-02)
|
|
4
|
+
|
|
5
|
+
### Issue #25: fix-empty-body injects `<p> </p>`, not bare ` `
|
|
6
|
+
|
|
7
|
+
Space Prison (Calibre book #11556, the Pyramid 1960 scan-conversion rip)
|
|
8
|
+
surfaced the defect in a phase-3 sweep: the book's 205-byte shell page
|
|
9
|
+
(`OEBPS/text00001.html`, `<body></body>` under an XHTML 1.0 Strict
|
|
10
|
+
DOCTYPE) made `--fix-empty-body` flip its own `--all` candidate from a
|
|
11
|
+
clean decrease (`0f/6e/0w -> 0f/5e/0w` core) into a regression the gate
|
|
12
|
+
had to refuse (`0f/6e/0w -> 1f/4e/0w --all`), the delta exactly
|
|
13
|
+
`fix_empty_body:1` and the new count a FATAL RSC-016, "The entity nbsp
|
|
14
|
+
was referenced, but not declared".
|
|
15
|
+
|
|
16
|
+
The root cause has two halves, both verified against epubcheck 5.3 with
|
|
17
|
+
a minimal-EPUB matrix across XHTML 1.0 Strict / Transitional / XHTML
|
|
18
|
+
1.1 / no-DOCTYPE shapes on EPUB2 packages and plain `<!DOCTYPE html>`
|
|
19
|
+
on EPUB3:
|
|
20
|
+
|
|
21
|
+
- epubcheck validates EPUB2 content docs against the XHTML 1.1 DTD and
|
|
22
|
+
resolves named entities only from DTDs it carries locally (the XHTML
|
|
23
|
+
1.1 one). ` ` is not one of XML's five predefined entities, so
|
|
24
|
+
under every DOCTYPE epubcheck does not resolve locally -- XHTML 1.0
|
|
25
|
+
Strict and Transitional, and no DOCTYPE at all -- the injection was
|
|
26
|
+
an undeclared-entity parse fatal. EPUB3 is no refuge: XHTML content
|
|
27
|
+
documents are validated as XML there too, and ` ` is fatal
|
|
28
|
+
under every DOCTYPE shape tested.
|
|
29
|
+
- The XHTML 1.1 body content model is block elements only, so bare
|
|
30
|
+
character data (named or numeric) never satisfied it anywhere: the
|
|
31
|
+
error the fix was named for ("element body incomplete") never
|
|
32
|
+
cleared. The old injection therefore never once applied under
|
|
33
|
+
epubcheck 5.x -- every run was either a fatal-regression reject or
|
|
34
|
+
an equal-noop -- which is why hardening it breaks no user: there was
|
|
35
|
+
no working behavior to preserve, only a dead flag whose only live
|
|
36
|
+
behavior was flipping candidates into rejects.
|
|
37
|
+
|
|
38
|
+
The injection is now the exact literal `<p> </p>`. The numeric
|
|
39
|
+
character reference needs no DTD in any validation path, and the
|
|
40
|
+
wrapping `<p>` is the block element the content model demands; the
|
|
41
|
+
matrix shows it clearing "body incomplete" on every EPUB2 DOCTYPE
|
|
42
|
+
shape with zero fatals and zero new errors, while on EPUB3 books
|
|
43
|
+
(where an empty body raises nothing) the gate answers equal and
|
|
44
|
+
nothing applies. The gate now accepts the fixture honestly:
|
|
45
|
+
`0f/6e/0w -> 0f/4e/0w` (body incomplete and the NCX playOrder error
|
|
46
|
+
both fixed; the book's four pre-existing error-nits stay), applied
|
|
47
|
+
under `--all`.
|
|
48
|
+
|
|
49
|
+
The flag's name, selection rule, opt-in status, and count key are
|
|
50
|
+
unchanged; only the injected bytes differ. `--fix-empty-body` and the
|
|
51
|
+
plugin's default pass never overlapped (the plugin runs the default
|
|
52
|
+
pass only), so the Bindery Repair zip in this release just carries the
|
|
53
|
+
updated transform. Tests: `TestFixEmptyBody` (test_transforms.py) pins
|
|
54
|
+
the literal, the no-named-entity invariant, whitespace/attribute/case
|
|
55
|
+
shapes, and untouched non-empty bodies; test_epub.py carries the
|
|
56
|
+
issue #25 regression shape (a Strict-DOCTYPE shell doc whose candidate
|
|
57
|
+
must show the numeric-in-block injection and never ` `). Suite:
|
|
58
|
+
832 tests.
|
|
59
|
+
|
|
60
|
+
## v0.60.0 (2026-11-01)
|
|
61
|
+
|
|
62
|
+
### Issue #24: the oracle budget also scales on archive entry count
|
|
63
|
+
|
|
64
|
+
The Summa Theologica (Calibre book #7301: 10,440,245 bytes, 3,831
|
|
65
|
+
archive entries, 3,808 spine documents, a 16,456-finding playOrder NCX)
|
|
66
|
+
kept dying at the gate even after v0.55.0's size-scaled budget, and the
|
|
67
|
+
first miss was the units: the size term divides by binary MiB, and
|
|
68
|
+
10,440,245 bytes is 9.96 MiB, so the book every doc reports as 10.4 MB
|
|
69
|
+
computed the flat 300s base with no size chunk at all. Even with a size
|
|
70
|
+
chunk it would have missed: epubcheck measured 384s on the source and
|
|
71
|
+
445s on the `--all` candidate (both runs under sweep contention), because
|
|
72
|
+
the book's cost is entry-count-bound (3,831 entries, 3,808 spine
|
|
73
|
+
documents), not bytes-bound. Either way the retry burned the same way
|
|
74
|
+
and every sweep misfiled the book as `(epubcheck timeout)` -- it stayed
|
|
75
|
+
a candidate forever.
|
|
76
|
+
|
|
77
|
+
Two budget changes, same 1800s cap. The size term counts decimal MB
|
|
78
|
+
(how book sizes are actually reported), so exactly-10-MiB files earn
|
|
79
|
+
their chunk instead of straddling the threshold. And the budget gains an
|
|
80
|
+
entry term shaped like the size term at finer granularity: +60s per
|
|
81
|
+
started 500 archive entries above the first 1,000. The Summa computes
|
|
82
|
+
720s (300 base + 60 size + 360 entries) against its measured 445s worst
|
|
83
|
+
case; books at or under 1,000 entries and 10 MB keep their historical
|
|
84
|
+
budget exactly, and a path that does not open as a zip (the budget is
|
|
85
|
+
computed before the book is judged) falls back to the size term alone.
|
|
86
|
+
No gate or CLI surface changes; the sweep selectors and the OracleTimeout
|
|
87
|
+
contract are untouched, and no library dry-run re-validation is
|
|
88
|
+
triggered (the change is the oracle's patience, not an acceptance bar).
|
|
89
|
+
|
|
90
|
+
The diagnosis also settled what the repair buys: epubcheck DOES police
|
|
91
|
+
the playOrder contract (2 of the source's 6 RSC-005 error entries are
|
|
92
|
+
exactly that: identical playOrder values for different targets, and
|
|
93
|
+
playOrder sequence gaps), so the default-pass candidate measures
|
|
94
|
+
0f/6e -> 0f/4e, a strict error decrease the gate accepts; the scoped
|
|
95
|
+
`--all` candidate measures 0f/0e/0w. Measured pins live in
|
|
96
|
+
tests/test_validate.py: exactly 10 MiB earns its decimal chunk (360s),
|
|
97
|
+
the 3,831-entry shape at <= 10 MB computes 660s, the full Summa shape
|
|
98
|
+
(10.44 decimal MB + 3,831 entries) computes 720s, exactly 1,000 entries
|
|
99
|
+
stays at the size budget, 1,001 entries adds the first chunk, and 2,500
|
|
100
|
+
entries compute 540s.
|
|
101
|
+
|
|
3
102
|
## v0.59.0 (2026-10-28)
|
|
4
103
|
|
|
5
104
|
### Issue #23: the dangling NCX navPoint, pruned and re-sequenced in one gate
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "bindery-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.61.0"
|
|
8
8
|
description = "Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
classifiers = [
|
|
@@ -53,6 +53,7 @@ The plugin vendors the core and never gates (the epubcheck latency ruling).
|
|
|
53
53
|
| 0.57 | The embed scar's element half and the end of the silent equal (issue #22): `--fix-unbound-prefix` now binds an unbound ELEMENT prefix inline (the 9.7.0 writer self-declares xmlns:dc on every dc: element but one; rule 2 for a literal dc: element resolves to the DC namespace itself; Hackers 1f/1e -> 0f/1e with both binds in one run, and the half-repaired intermediate that read equal now repairs), and the masked-axis reclassification generalizes from the v0.51.0 NCX sync to every fired fix: a gate noop on a book that started and stayed fatal reads partial (the manual list) instead of a silent equal; the specimen's residual ncx:-prefixed dtb:uid gap shipped as the v0.57.1 patch (both matchers read a prefixed `<ncx:meta>` carrier; Hackers fully clean 0f/0e/0w) | v0.57.0-v0.57.1 |
|
|
54
54
|
| 0.58 | The structural dry run's four rejects (issues #18-#21), each bisected to the root on its specimens: the private-ZipInfo write contract (issue #18: writestr stamps the info it is handed with the DESTINATION offsets, one shared info repointed zin's own lookups into the output file -- all 57 "unreadable" books, the "possible zip bomb" included, were this one bug; the peek also catches BadZipFile per the issue), bracket-encoded renamed references plus the start-tag/PI anchor on both reference rewriters (issue #19: raw brackets are RSC-020 path-illegal -- IJ 108e -> 22e -- and the unanchored regex corrupted display text that teaches HTML -- Hacking's two RSC-016 fatals), the entity-decoding href resolver and prefix-tolerant OPF matchers (issue #20: `&` hrefs vs raw-`&` entries made prune and declare swap the same items on the D&D four, and `<opf:item>` blindness duplicated whole manifests on the Sony three), and the three #21 mechanisms (guide-fragment strip, the post-anchor playOrder re-run, the comment-blind duplicate-id rename) | v0.58.0 |
|
|
55
55
|
| 0.59 | The dangling-NCX prune (issue #23): `--prune-dangling-ncx-targets` removes navPoint/pageTarget elements whose `<content src>` names a document the archive never contained (RSC-007 the manifest prune cannot reach: no item exists for a file the OPF never declared), node-exact over a balanced stack scanner (dead subtree drops as one span, dead parent spliced above live children, scheme/bare-fragment/comment-hidden/contentless refused, unbalanced walk aborts), with the contract-gated playOrder pass re-run in the same branch -- the hand-surgery-plus-second-run rescue is one gate (fixture 0f/1e/0w -> 0f/0e/0w: one navPoint pruned, 389 playOrder values re-sequenced) | v0.59.0 |
|
|
56
|
+
| 0.60 | The oracle budget scales on decimal MB and archive entry count (issue #24): the Summa Theologica (10,440,245 bytes = 9.96 binary MiB, 3,831 entries, 3,808 spine docs, 16,456 playOrder findings) is entry-count-bound, not bytes-bound -- measured 384s source and 445s candidate against the flat 300s its bytes computed (the binary 10 MiB divisor ate the size chunk; the book the docs call 10.4 MB never earned one) -- so every sweep double-timed-out and misfiled it as `(epubcheck timeout)`; the size term counts decimal MB now, the entry term adds +60s per started 500 entries above 1,000 (same 1800s cap), the book computes 720s, and a non-zip path falls back to size-only. epubcheck does police the playOrder contract (2 of the source's 6 RSC-005 error entries): default-pass candidate measures 0f/6e -> 0f/4e (accept), scoped `--all` candidate 0f/0e/0w | v0.60.0 |
|
|
56
57
|
|
|
57
58
|
## Open work
|
|
58
59
|
|
|
@@ -150,8 +150,12 @@ byte-for-byte. The normal gate applies.
|
|
|
150
150
|
Twenty-six repairs go past well-formedness and therefore require their own flag; none is ever
|
|
151
151
|
part of the default pipeline:
|
|
152
152
|
|
|
153
|
-
- **`--fix-empty-body`**:
|
|
154
|
-
incomplete").
|
|
153
|
+
- **`--fix-empty-body`**: `<p> </p>` inside a strictly empty `<body></body>` ("body
|
|
154
|
+
incomplete"). The numeric character reference needs no DTD and the wrapping `p` is the
|
|
155
|
+
block element XHTML 1.1's body content model demands (issue #25: the ` ` this fix
|
|
156
|
+
originally injected was an undeclared-entity parse fatal under every DOCTYPE epubcheck
|
|
157
|
+
does not resolve locally, and bare character data never satisfied the content model).
|
|
158
|
+
Adds visible content, hence opt-in like `--add-img-alt`.
|
|
155
159
|
- **`--fix-missing-title`**: inject `<title>Unknown</title>` when the head has no usable
|
|
156
160
|
title.
|
|
157
161
|
- **`--fix-id-colons`**: illegal colons in `id="X:Y"` and their matching internal
|
|
@@ -11,7 +11,7 @@ Calibre plugin built from this core.
|
|
|
11
11
|
Single source of truth for the version. pyproject.toml must match VERSION below.
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
|
-
VERSION = "0.
|
|
14
|
+
VERSION = "0.61.0"
|
|
15
15
|
__version__ = VERSION
|
|
16
16
|
|
|
17
17
|
from .epub import RepairReport, repair_epub # noqa: E402
|
|
@@ -1668,10 +1668,14 @@ _REPAIR_FLAG_TABLE: dict[str, list[tuple[str, str, str]]] = {
|
|
|
1668
1668
|
"structural repairs: markup and nesting": [
|
|
1669
1669
|
(
|
|
1670
1670
|
"fix_empty_body",
|
|
1671
|
-
"
|
|
1672
|
-
"
|
|
1673
|
-
"incomplete' error).
|
|
1674
|
-
"
|
|
1671
|
+
"fill a strictly empty <body></body> with <p> </p>",
|
|
1672
|
+
"inject <p> </p> inside a strictly empty <body></body> (the "
|
|
1673
|
+
"'body incomplete' error). The numeric character reference needs "
|
|
1674
|
+
"no DTD (a named is an undeclared-entity parse fatal under "
|
|
1675
|
+
"every DOCTYPE epubcheck does not resolve locally) and the "
|
|
1676
|
+
"wrapping p is the block element XHTML 1.1's body content model "
|
|
1677
|
+
"demands, which bare text never satisfied. Adds visible content "
|
|
1678
|
+
"the author never wrote; hence opt-in.",
|
|
1675
1679
|
),
|
|
1676
1680
|
(
|
|
1677
1681
|
"fix_missing_title",
|
|
@@ -1994,7 +1994,10 @@ def repair_epub(
|
|
|
1994
1994
|
With `strip_watermarks`, remove known producer watermarks (e.g. OceanofPDF).
|
|
1995
1995
|
|
|
1996
1996
|
The structural repairs are all opt-in, never part of the core pass:
|
|
1997
|
-
with `empty_body`,
|
|
1997
|
+
with `empty_body`, inject the exact literal `<p> </p>` into a strictly
|
|
1998
|
+
empty <body> (the numeric reference needs no DTD and the wrapping p is
|
|
1999
|
+
the block epubcheck's XHTML 1.1 body content model demands — issue #25);
|
|
2000
|
+
with `missing_title`,
|
|
1998
2001
|
inject a <title>Unknown</title> fallback; with `id_colons`, translate illegal
|
|
1999
2002
|
colons in id attributes and their #fragment references; with `block_in_inline`,
|
|
2000
2003
|
unwrap a <span> that illegally wraps a block element; with `invalid_value`, strip
|
|
@@ -631,7 +631,19 @@ def unwrap_illegal_tags(
|
|
|
631
631
|
|
|
632
632
|
@_outside_protected
|
|
633
633
|
def fix_empty_body(s: str) -> tuple[str, int]:
|
|
634
|
-
"""
|
|
634
|
+
"""Give a strictly empty `<body></body>` one minimal paragraph, the exact
|
|
635
|
+
literal `<p> </p>` ("body incomplete").
|
|
636
|
+
|
|
637
|
+
That literal, not the bare ` ` this fix shipped with (issue #25):
|
|
638
|
+
epubcheck validates EPUB2 content docs against the
|
|
639
|
+
with (issue #25): epubcheck validates EPUB2 content docs against the
|
|
640
|
+
XHTML 1.1 DTD, whose body content model is block elements only, so bare
|
|
641
|
+
character data never cleared the error, and ` ` is not one of XML's
|
|
642
|
+
five predefined entities, so under every DOCTYPE whose DTD epubcheck does
|
|
643
|
+
not load locally the injection was itself a parse-fatal RSC-016 ("The
|
|
644
|
+
entity "nbsp" was referenced, but not declared"). The numeric
|
|
645
|
+
character reference needs no DTD at all, and the wrapping `<p>` is the
|
|
646
|
+
block the content model demands.
|
|
635
647
|
|
|
636
648
|
Opt-in (--fix-empty-body): this ADDS visible content the author never wrote,
|
|
637
649
|
the same reason --add-img-alt is opt-in; it is never part of the core pass.
|
|
@@ -641,7 +653,7 @@ def fix_empty_body(s: str) -> tuple[str, int]:
|
|
|
641
653
|
def repl(m: re.Match) -> str:
|
|
642
654
|
nonlocal count
|
|
643
655
|
count += 1
|
|
644
|
-
return m.group(1) + "
|
|
656
|
+
return m.group(1) + "<p> </p>" + m.group(2)
|
|
645
657
|
|
|
646
658
|
s, n = re.subn(r"(<body[^>]*>\s*)(</body>)", repl, s, flags=re.IGNORECASE)
|
|
647
659
|
return s, n
|
|
@@ -22,6 +22,7 @@ import subprocess
|
|
|
22
22
|
import tempfile
|
|
23
23
|
import threading
|
|
24
24
|
import time
|
|
25
|
+
import zipfile
|
|
25
26
|
from dataclasses import dataclass
|
|
26
27
|
from pathlib import Path
|
|
27
28
|
|
|
@@ -52,22 +53,47 @@ class OracleTimeout(RuntimeError):
|
|
|
52
53
|
|
|
53
54
|
def oracle_timeout_for(path: Path, base: int = 300) -> int:
|
|
54
55
|
"""The epubcheck budget for one book: the base, plus a minute per
|
|
55
|
-
started 10 MB above 10 MB,
|
|
56
|
+
started 10 MB above 10 MB, plus a minute per started 500 archive
|
|
57
|
+
entries above 1,000, capped at 30 minutes (issues #15, #24).
|
|
56
58
|
|
|
57
59
|
The 2026-10-24 whole-library runs hit a 300s ceiling on the Summa
|
|
58
60
|
Theologica (10.4 MB, 3,795-playOrder NCX): it timed out in one run and
|
|
59
61
|
passed the identical gate in another, so the fixed budget misfiled a
|
|
60
62
|
timing flake as a repair error. Bigger books simply get more time;
|
|
61
|
-
|
|
63
|
+
the historical 300s base itself never changes.
|
|
64
|
+
|
|
65
|
+
The size term alone still misfiled the Summa twice (issue #24). First
|
|
66
|
+
the units: 10,440,245 bytes is 9.96 MiB, just under the binary 10 MiB
|
|
67
|
+
threshold, so the book the docs call 10.4 MB computed the flat 300s
|
|
68
|
+
base with no size chunk at all; the term now counts decimal MB,
|
|
69
|
+
matching how book sizes are actually reported. Second the shape:
|
|
70
|
+
measured at 384s on the source and 445s on the candidate, the book is
|
|
71
|
+
entry-count-bound (3,831 archive entries, 3,808 spine documents), not
|
|
72
|
+
bytes-bound. The entry term mirrors the size term's started-chunk
|
|
73
|
+
shape at finer granularity: a minute per started 500 entries above
|
|
74
|
+
the first 1,000 (an exact 500-multiple earns its chunk, like the
|
|
75
|
+
size term's exact-10-MB multiples), so ordinary books keep their
|
|
76
|
+
historical budget exactly. A path that does not open as a zip (the
|
|
77
|
+
budget is computed before the book is judged) falls back to the size
|
|
78
|
+
term alone.
|
|
62
79
|
"""
|
|
63
80
|
try:
|
|
64
|
-
size_mb = path.stat().st_size /
|
|
81
|
+
size_mb = path.stat().st_size / 10**6
|
|
65
82
|
except OSError:
|
|
66
83
|
return base
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
84
|
+
timeout = base
|
|
85
|
+
if size_mb > 10:
|
|
86
|
+
chunks = int((size_mb - 10) // 10) + 1
|
|
87
|
+
timeout += 60 * chunks
|
|
88
|
+
try:
|
|
89
|
+
with zipfile.ZipFile(path) as zf:
|
|
90
|
+
entries = len(zf.namelist())
|
|
91
|
+
except (OSError, zipfile.BadZipFile):
|
|
92
|
+
entries = 0
|
|
93
|
+
if entries > 1000:
|
|
94
|
+
entry_chunks = ((entries - 1000) // 500) + 1
|
|
95
|
+
timeout += 60 * entry_chunks
|
|
96
|
+
return min(timeout, 1800)
|
|
71
97
|
|
|
72
98
|
|
|
73
99
|
def epubcheck_available() -> bool:
|
|
@@ -1012,6 +1012,28 @@ class TestOptInStructuralRepairs(unittest.TestCase):
|
|
|
1012
1012
|
report = repair_epub(self.src, self.dst, RepairFlags(empty_body=True))
|
|
1013
1013
|
self.assertIn("fix_empty_body", report.fixes)
|
|
1014
1014
|
|
|
1015
|
+
def test_empty_body_injection_is_numeric_in_a_block(self):
|
|
1016
|
+
# issue #25 (Space Prison #11556): the shell doc declares XHTML 1.0
|
|
1017
|
+
# Strict, whose DTD epubcheck does not resolve locally, so the old
|
|
1018
|
+
# bare ` ` injection was an undeclared-entity parse fatal
|
|
1019
|
+
# (RSC-016) and the candidate died in the gate as a regression. The
|
|
1020
|
+
# injection must be the DTD-independent numeric reference inside the
|
|
1021
|
+
# block element XHTML 1.1's body content model demands.
|
|
1022
|
+
self._build(
|
|
1023
|
+
css="p { margin: 0 }",
|
|
1024
|
+
content=(
|
|
1025
|
+
'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" '
|
|
1026
|
+
'"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">'
|
|
1027
|
+
'<html xmlns="http://www.w3.org/1999/xhtml">\r'
|
|
1028
|
+
"<head><title></title>\r</head>\r<body></body>\r</html>\r"
|
|
1029
|
+
),
|
|
1030
|
+
)
|
|
1031
|
+
repair_epub(self.src, self.dst, RepairFlags(empty_body=True))
|
|
1032
|
+
with zipfile.ZipFile(self.dst) as z:
|
|
1033
|
+
c = z.read("OEBPS/c1.xhtml").decode()
|
|
1034
|
+
self.assertIn("<body><p> </p></body>", c)
|
|
1035
|
+
self.assertNotIn(" ", c)
|
|
1036
|
+
|
|
1015
1037
|
|
|
1016
1038
|
class TestPruneMissingResources(unittest.TestCase):
|
|
1017
1039
|
"""--prune-missing-resources: references to files the archive does not
|
|
@@ -550,6 +550,53 @@ class TestStructuralRepairsAreOptIn(unittest.TestCase):
|
|
|
550
550
|
self.assertGreaterEqual(n, 1)
|
|
551
551
|
|
|
552
552
|
|
|
553
|
+
class TestFixEmptyBody(unittest.TestCase):
|
|
554
|
+
"""The injection is `<p> </p>` (issue #25): epubcheck validates EPUB2
|
|
555
|
+
content docs against the XHTML 1.1 DTD, whose body content model is block
|
|
556
|
+
elements only, so the bare ` ` this fix shipped with never cleared
|
|
557
|
+
the error and was itself an undeclared-entity parse fatal (RSC-016) under
|
|
558
|
+
every DOCTYPE epubcheck does not resolve locally."""
|
|
559
|
+
|
|
560
|
+
def test_injects_numeric_entity_in_a_block(self):
|
|
561
|
+
out, n = fix_empty_body("<html><body></body></html>")
|
|
562
|
+
self.assertEqual(n, 1)
|
|
563
|
+
self.assertEqual(out, "<html><body><p> </p></body></html>")
|
|
564
|
+
|
|
565
|
+
def test_never_emits_the_named_entity(self):
|
|
566
|
+
# the issue #25 regression class: is not one of XML's five
|
|
567
|
+
# predefined entities, so it is a parse fatal under a DOCTYPE whose
|
|
568
|
+
# DTD declares no HTML entity set
|
|
569
|
+
out, _ = fix_empty_body("<body></body>")
|
|
570
|
+
self.assertNotIn(" ", out)
|
|
571
|
+
|
|
572
|
+
def test_whitespace_only_body_fires(self):
|
|
573
|
+
out, n = fix_empty_body("<body>\r</body>")
|
|
574
|
+
self.assertEqual(n, 1)
|
|
575
|
+
self.assertEqual(out, "<body>\r<p> </p></body>")
|
|
576
|
+
|
|
577
|
+
def test_body_attributes_and_leading_space_survive(self):
|
|
578
|
+
out, n = fix_empty_body('<body class="chapter">\n</body>')
|
|
579
|
+
self.assertEqual(n, 1)
|
|
580
|
+
self.assertEqual(out, '<body class="chapter">\n<p> </p></body>')
|
|
581
|
+
|
|
582
|
+
def test_case_insensitive_match(self):
|
|
583
|
+
out, n = fix_empty_body("<BODY></BODY>")
|
|
584
|
+
self.assertEqual(n, 1)
|
|
585
|
+
self.assertEqual(out, "<BODY><p> </p></BODY>")
|
|
586
|
+
|
|
587
|
+
def test_non_empty_body_untouched(self):
|
|
588
|
+
for body in ("<body><p>x</p></body>", '<body><img src="a.png"/></body>'):
|
|
589
|
+
with self.subTest(body=body):
|
|
590
|
+
out, n = fix_empty_body(body)
|
|
591
|
+
self.assertEqual((out, n), (body, 0))
|
|
592
|
+
|
|
593
|
+
def test_multiple_empty_bodies_count_individually(self):
|
|
594
|
+
# malformed (one body per document), but the count must stay honest
|
|
595
|
+
out, n = fix_empty_body("<body></body><body></body>")
|
|
596
|
+
self.assertEqual(n, 2)
|
|
597
|
+
self.assertEqual(out.count("<p> </p>"), 2)
|
|
598
|
+
|
|
599
|
+
|
|
553
600
|
class TestFixNcxPlayorder(unittest.TestCase):
|
|
554
601
|
def test_resequences_navpoint_attributes(self):
|
|
555
602
|
ncx = (
|
|
@@ -211,14 +211,19 @@ class FailureModes(unittest.TestCase):
|
|
|
211
211
|
|
|
212
212
|
|
|
213
213
|
class ScaledTimeout(unittest.TestCase):
|
|
214
|
-
"""The size-scaled default budget (
|
|
215
|
-
|
|
216
|
-
|
|
214
|
+
"""The size- and entry-scaled default budget (issues #15, #24): the
|
|
215
|
+
Summa Theologica flaked at the fixed 300s (issue #15), then outgrew
|
|
216
|
+
the size term itself -- 384s source and 445s candidate against the
|
|
217
|
+
flat 300s its bytes computed (9.96 binary MiB never earned a chunk) --
|
|
218
|
+
because epubcheck's cost on it is entry-count-bound (3,831 entries,
|
|
219
|
+
3,808 spine docs). Books over 10 decimal MB get a minute per started
|
|
220
|
+
10 MB; books over 1,000 entries get a minute per started 500 entries;
|
|
221
|
+
capped at 30 minutes."""
|
|
217
222
|
|
|
218
223
|
def test_at_or_under_ten_mb_keeps_the_base(self) -> None:
|
|
219
224
|
with tempfile.TemporaryDirectory() as td:
|
|
220
225
|
small = Path(td) / "small.epub"
|
|
221
|
-
small.write_bytes(b"x" *
|
|
226
|
+
small.write_bytes(b"x" * 9_500_000)
|
|
222
227
|
self.assertEqual(oracle_timeout_for(small), 300)
|
|
223
228
|
empty = Path(td) / "missing.epub"
|
|
224
229
|
self.assertEqual(oracle_timeout_for(empty), 300)
|
|
@@ -226,12 +231,65 @@ class ScaledTimeout(unittest.TestCase):
|
|
|
226
231
|
def test_over_ten_mb_scales_per_started_chunk(self) -> None:
|
|
227
232
|
with tempfile.TemporaryDirectory() as td:
|
|
228
233
|
book = Path(td) / "summa.epub"
|
|
229
|
-
book.write_bytes(b"x" *
|
|
234
|
+
book.write_bytes(b"x" * 10_400_000)
|
|
230
235
|
self.assertEqual(oracle_timeout_for(book), 360)
|
|
231
236
|
big = Path(td) / "big.epub"
|
|
232
|
-
big.write_bytes(b"x" *
|
|
237
|
+
big.write_bytes(b"x" * 55_000_000)
|
|
233
238
|
self.assertEqual(oracle_timeout_for(big), 600)
|
|
234
239
|
|
|
240
|
+
def test_binary_mib_trap_earns_its_decimal_chunk(self) -> None:
|
|
241
|
+
"""The Summa trap (issue #24): exactly 10 MiB is 10.49 decimal MB,
|
|
242
|
+
so the size term fires where the old binary divisor read it as
|
|
243
|
+
at-threshold and gave the flat 300s."""
|
|
244
|
+
with tempfile.TemporaryDirectory() as td:
|
|
245
|
+
book = Path(td) / "exactly-10mib.epub"
|
|
246
|
+
book.write_bytes(b"x" * (10 * 2**20))
|
|
247
|
+
self.assertEqual(oracle_timeout_for(book), 360)
|
|
248
|
+
|
|
249
|
+
def test_entry_heavy_book_scales_per_started_half_thousand(self) -> None:
|
|
250
|
+
"""The Summa entry shape (issue #24): 3,831 tiny entries at
|
|
251
|
+
<= 10 decimal MB compute 300 + 6 x 60 = 660s; the entry term
|
|
252
|
+
mirrors the size term's started-chunk shape at 500-entry
|
|
253
|
+
granularity."""
|
|
254
|
+
with tempfile.TemporaryDirectory() as td:
|
|
255
|
+
book = Path(td) / "summa-shape.epub"
|
|
256
|
+
with zipfile.ZipFile(book, "w") as zf:
|
|
257
|
+
for i in range(3831):
|
|
258
|
+
zf.writestr(f"OEBPS/Text/c{i}.xhtml", "<html/>")
|
|
259
|
+
self.assertEqual(oracle_timeout_for(book), 660)
|
|
260
|
+
|
|
261
|
+
def test_entry_term_boundaries(self) -> None:
|
|
262
|
+
with tempfile.TemporaryDirectory() as td:
|
|
263
|
+
exactly = Path(td) / "exactly-1000.epub"
|
|
264
|
+
with zipfile.ZipFile(exactly, "w") as zf:
|
|
265
|
+
for i in range(1000):
|
|
266
|
+
zf.writestr(f"c{i}.xhtml", "<html/>")
|
|
267
|
+
self.assertEqual(oracle_timeout_for(exactly), 300)
|
|
268
|
+
over = Path(td) / "1001.epub"
|
|
269
|
+
with zipfile.ZipFile(over, "w") as zf:
|
|
270
|
+
for i in range(1001):
|
|
271
|
+
zf.writestr(f"c{i}.xhtml", "<html/>")
|
|
272
|
+
self.assertEqual(oracle_timeout_for(over), 360)
|
|
273
|
+
wide = Path(td) / "2500.epub"
|
|
274
|
+
with zipfile.ZipFile(wide, "w") as zf:
|
|
275
|
+
for i in range(2500):
|
|
276
|
+
zf.writestr(f"c{i}.xhtml", "<html/>")
|
|
277
|
+
self.assertEqual(oracle_timeout_for(wide), 540)
|
|
278
|
+
|
|
279
|
+
def test_the_real_summa_shape(self) -> None:
|
|
280
|
+
"""The actual book (issue #24): 10,440,245 bytes (10.44 decimal
|
|
281
|
+
MB, one size chunk) plus 3,831 entries (six entry chunks) compute
|
|
282
|
+
300 + 60 + 360 = 720s against its measured 384s source / 445s
|
|
283
|
+
candidate."""
|
|
284
|
+
with tempfile.TemporaryDirectory() as td:
|
|
285
|
+
book = Path(td) / "summa-real.epub"
|
|
286
|
+
payload = b"x" * 10_200_000
|
|
287
|
+
with zipfile.ZipFile(book, "w") as zf:
|
|
288
|
+
zf.writestr("OEBPS/big.xhtml", payload)
|
|
289
|
+
for i in range(3830):
|
|
290
|
+
zf.writestr(f"OEBPS/Text/c{i}.xhtml", "<html/>")
|
|
291
|
+
self.assertEqual(oracle_timeout_for(book), 720)
|
|
292
|
+
|
|
235
293
|
def test_cap(self) -> None:
|
|
236
294
|
with tempfile.TemporaryDirectory() as td:
|
|
237
295
|
huge = Path(td) / "huge.epub"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|