bindery-cli 0.61.1__tar.gz → 0.62.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/PKG-INFO +1 -1
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/patchnotes.md +76 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/pyproject.toml +1 -1
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/roadmap.md +1 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/__init__.py +1 -1
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/cli.py +57 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/epub.py +347 -2
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/transforms.py +30 -10
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_cli.py +20 -7
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_epub.py +485 -11
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_flags_real_core.py +85 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_flags_wiring.py +4 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_transforms.py +74 -2
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/uv.lock +1 -1
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/.github/FUNDING.yml +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/.github/workflows/ci.yml +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/.github/workflows/publish.yml +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/.gitignore +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/AGENTS.md +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/LICENSE +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/OBSERVE-RESEARCH.md +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/README.md +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/docs/screenshots/library-sweep.png +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/logo.svg +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/plugin/__init__.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/project.done +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/run_tests.sh +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/scripts/FastSweep.java +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/scripts/build_plugin.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/scripts/fast_sweep.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/spec.md +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/__main__.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/audit.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/library.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/observe.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/pagination.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/reserialize.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/validate.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/src/bindery/watermark.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_audit.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_comment_hyphens.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_contamination.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_cover_tocdrift.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_help.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_library.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_matcher_hardening.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_observe.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_pagination.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_plugin.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_reserialize.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_space_renames.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_spine_guard.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_validate.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_version.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_watermark.py +0 -0
- {bindery_cli-0.61.1 → bindery_cli-0.62.0}/tests/test_xml_hardening.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bindery-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.62.0
|
|
4
4
|
Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
|
|
5
5
|
Project-URL: Homepage, https://github.com/VirInvictus/bindery-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/VirInvictus/bindery-cli
|
|
@@ -1,5 +1,81 @@
|
|
|
1
1
|
# bindery-cli Patch Notes
|
|
2
2
|
|
|
3
|
+
## v0.62.0 (2026-11-03)
|
|
4
|
+
|
|
5
|
+
### The observatory batch: three predicate repairs and four new flags
|
|
6
|
+
|
|
7
|
+
The 2026-11-03 whole-library observatory run answered the default pass's
|
|
8
|
+
`0/0/0` (nothing applies, nothing rejects) with two discoveries: twelve
|
|
9
|
+
census codes had never been triaged, and the full `--all` arsenal's 322
|
|
10
|
+
rejections were 283 bundles poisoned by two transforms whose firing
|
|
11
|
+
predicates outran what epubcheck actually polices. This release carries the
|
|
12
|
+
batch that came out of it, every piece verified against epubcheck 5.3 on
|
|
13
|
+
real specimens or minimal matrices before the code moved.
|
|
14
|
+
|
|
15
|
+
**The three predicate repairs** (the playOrder precedent repeating):
|
|
16
|
+
|
|
17
|
+
- `fix_ncx_playorder`'s contract demanded values 1..K; the oracle polices
|
|
18
|
+
sharing, distinctness, and gaps and never the base (Age of Innocence
|
|
19
|
+
validates clean at 0..43; singleton-2, pair-2-3, and offset-5-8 verified).
|
|
20
|
+
The clause is base-free contiguous now, silencing the 107 no-op
|
|
21
|
+
renumberings the observatory measured while keeping every real fix.
|
|
22
|
+
- `fix_pagelist_class` inserts the collision-checked id alongside the
|
|
23
|
+
class: class-only insertion flipped epubcheck into demanding the id on
|
|
24
|
+
the element that had just gained the class (Feel Free: bare pageList
|
|
25
|
+
validates clean, `class="pages"` alone draws RSC-005 'missing required
|
|
26
|
+
attribute "id"', id+class validates fully clean) -- the 215-rejection
|
|
27
|
+
poison. The books that used to regress now measure flat; the ones that
|
|
28
|
+
used to accept still do.
|
|
29
|
+
- `fix_missing_title` fills the self-closed `<title/>`: the paired-fill
|
|
30
|
+
branch was unreachable dead code behind the no-op check, so every
|
|
31
|
+
self-closed shape fell through to injection beside the standing tag
|
|
32
|
+
(the 70-rejection poison; 84 sampled reject docs, all exactly
|
|
33
|
+
`<title/>`, each gaining RSC-005 'element "title" not allowed here').
|
|
34
|
+
|
|
35
|
+
**The four new flags** (the never-triaged census lanes):
|
|
36
|
+
|
|
37
|
+
- `--fix-package-version` (OPF-001, 26 books): rewrites the literal
|
|
38
|
+
`version="1.0"` on the package element to `"2.0"` -- an EPUB version
|
|
39
|
+
epubcheck cannot parse. Refuses everything else: the OEB-1.0-namespace
|
|
40
|
+
package with no version attribute is the designed refusal, and any other
|
|
41
|
+
spelling would be a guess about the producer's intent. Audition
|
|
42
|
+
validates fully clean after the bump.
|
|
43
|
+
- `--fix-doctype` (HTM-004, 135 books): replaces a content document's
|
|
44
|
+
DOCTYPE when it carries a public identifier its package profile forbids
|
|
45
|
+
-- EPUB 3 requires any DOCTYPE to be `<!DOCTYPE html>` (an XHTML 1.x
|
|
46
|
+
public doctype is a spec violation), and EPUB 2's OPS content docs are
|
|
47
|
+
XHTML 1.1 (the Hound-of-the-Baskervilles shape: found 1.0 Strict,
|
|
48
|
+
expected 1.1). Both census shapes verified on facility copies. The
|
|
49
|
+
first build also inserted a DOCTYPE when absent; the whole-library
|
|
50
|
+
firing census measured that flat on ~2,400 of 2,545 firing books -- an
|
|
51
|
+
absent DOCTYPE is legal in both profiles and the oracle tolerates it --
|
|
52
|
+
so insertion was removed before release: the flag fires only where the
|
|
53
|
+
declaration exists and is wrong. A whitespace-variant of the expected
|
|
54
|
+
form is a no-op, and a DOCTYPE carrying an internal subset is never
|
|
55
|
+
touched (the subset declares entities).
|
|
56
|
+
- `--prune-invalid-guide-refs` (OPF-032, 125 books): removes guide
|
|
57
|
+
`<reference>` elements whose target is declared but is not an OPS
|
|
58
|
+
content document -- the Random House `msr` guide block pointing
|
|
59
|
+
cover/thumbimage references at JPEGs is the dominant fingerprint. The
|
|
60
|
+
guide is furniture, so removal is content-neutral; a reference to an
|
|
61
|
+
undeclared file stays OPF-031 (declare-missing's lane), as do external
|
|
62
|
+
schemes and paired elements. Widdershins validates fully clean after
|
|
63
|
+
the prune.
|
|
64
|
+
- `--fix-item-properties` (OPF-014, 74 books): a whole-book census reads
|
|
65
|
+
what each content document contains and merges the tokens into the
|
|
66
|
+
owning manifest item's `properties` attribute, sorted and deduplicated.
|
|
67
|
+
`scripted` from a `<script>` element, an on* handler attribute, or a
|
|
68
|
+
`<form>` -- epubcheck 5.3 demands scripted for form documents, proven
|
|
69
|
+
by tag-off bisection on the Rustonomicon's scraped GitHub page; `svg`
|
|
70
|
+
from an `<svg>` element; `remote-resources` from an absolute-scheme src
|
|
71
|
+
attribute, a remote `<link href>`, or a remote CSS url(). EPUB 3
|
|
72
|
+
packages only: the matrix proved declaring `properties` on an EPUB2
|
|
73
|
+
item draws the illegal-attribute finding while its OPF-014 demand
|
|
74
|
+
stands, so the EPUB2 carriers refuse and stay measured. The census's
|
|
75
|
+
own lane specimen (Rustonomicon) cleared its OPF-014 live.
|
|
76
|
+
|
|
77
|
+
Suite: 878 tests.
|
|
78
|
+
|
|
3
79
|
## v0.61.1 (2026-11-03)
|
|
4
80
|
|
|
5
81
|
### The hex-ref href crash, and the sweep that survives it
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "bindery-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.62.0"
|
|
8
8
|
description = "Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
classifiers = [
|
|
@@ -55,6 +55,7 @@ The plugin vendors the core and never gates (the epubcheck latency ruling).
|
|
|
55
55
|
| 0.59 | The dangling-NCX prune (issue #23): `--prune-dangling-ncx-targets` removes navPoint/pageTarget elements whose `<content src>` names a document the archive never contained (RSC-007 the manifest prune cannot reach: no item exists for a file the OPF never declared), node-exact over a balanced stack scanner (dead subtree drops as one span, dead parent spliced above live children, scheme/bare-fragment/comment-hidden/contentless refused, unbalanced walk aborts), with the contract-gated playOrder pass re-run in the same branch -- the hand-surgery-plus-second-run rescue is one gate (fixture 0f/1e/0w -> 0f/0e/0w: one navPoint pruned, 389 playOrder values re-sequenced) | v0.59.0 |
|
|
56
56
|
| 0.60 | The oracle budget scales on decimal MB and archive entry count (issue #24): the Summa Theologica (10,440,245 bytes = 9.96 binary MiB, 3,831 entries, 3,808 spine docs, 16,456 playOrder findings) is entry-count-bound, not bytes-bound -- measured 384s source and 445s candidate against the flat 300s its bytes computed (the binary 10 MiB divisor ate the size chunk; the book the docs call 10.4 MB never earned one) -- so every sweep double-timed-out and misfiled it as `(epubcheck timeout)`; the size term counts decimal MB now, the entry term adds +60s per started 500 entries above 1,000 (same 1800s cap), the book computes 720s, and a non-zip path falls back to size-only. epubcheck does police the playOrder contract (2 of the source's 6 RSC-005 error entries): default-pass candidate measures 0f/6e -> 0f/4e (accept), scoped `--all` candidate 0f/0e/0w | v0.60.0 |
|
|
57
57
|
| 0.61.1 | The hex-ref href crash and the sweep that survives it: `_NUMERIC_REF_RE` captured the hex ref's `x` into the digits group and `int('x2013', 16)` killed the first whole-library `--all` dry run (2026-11-03 observe lane; 36 hex-ref hrefs across 5 books, every one an external URL the decode reaches before the scheme check discards it; dormant since v0.58.0 because the resolver is opt-in-path-only); the fix strips the prefix before the int, and the sweep's repair loop gains the crash guard -- an unexpected per-book exception is that book's `error` verdict (`crash (Type): ...; not applied`), counted in the existing errors tally, instead of a run-killing traceback (the single-file verb keeps its traceback) | v0.61.1 |
|
|
58
|
+
| 0.62 | The observatory batch (the 2026-11-03 lane's answers): three predicate repairs -- the playOrder contract goes base-free (epubcheck polices sharing/distinctness/gaps, never the base; 107 no-op renumberings silenced), `fix_pagelist_class` inserts the collision-checked id beside the class (class-only made epubcheck demand the id it enabled; the 215-rejection poison), `fix_missing_title` fills the self-closed `<title/>` (the paired-fill branch was dead code; the 70-rejection poison) -- and four new flags for the never-triaged census codes: `--fix-package-version` (OPF-001; literal 1.0 only), `--fix-doctype` (HTM-004; present-but-forbidden declarations only, after the firing census killed insertion), `--prune-invalid-guide-refs` (OPF-032; declared-but-not-a-content-doc targets), `--fix-item-properties` (OPF-014; EPUB3-only, scripted/svg/remote-resources from a whole-book census) | v0.62.0 |
|
|
58
59
|
|
|
59
60
|
## Open work
|
|
60
61
|
|
|
@@ -11,7 +11,7 @@ Calibre plugin built from this core.
|
|
|
11
11
|
Single source of truth for the version. pyproject.toml must match VERSION below.
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
|
-
VERSION = "0.
|
|
14
|
+
VERSION = "0.62.0"
|
|
15
15
|
__version__ = VERSION
|
|
16
16
|
|
|
17
17
|
from .epub import RepairReport, repair_epub # noqa: E402
|
|
@@ -109,6 +109,10 @@ def _flags_from_args(args: argparse.Namespace) -> RepairFlags:
|
|
|
109
109
|
unterminated_attr=args.fix_unterminated_attr or all_,
|
|
110
110
|
unbound_prefix=args.fix_unbound_prefix or all_,
|
|
111
111
|
unique_identifier=args.fix_unique_identifier or all_,
|
|
112
|
+
package_version=args.fix_package_version or all_,
|
|
113
|
+
doctype=args.fix_doctype or all_,
|
|
114
|
+
prune_guide_refs=args.prune_invalid_guide_refs or all_,
|
|
115
|
+
item_properties=args.fix_item_properties or all_,
|
|
112
116
|
)
|
|
113
117
|
|
|
114
118
|
|
|
@@ -1977,6 +1981,59 @@ _REPAIR_FLAG_TABLE: dict[str, list[tuple[str, str, str]]] = {
|
|
|
1977
1981
|
"document untouched. Content documents are not touched (their "
|
|
1978
1982
|
"unbound prefixes have the --strip-bad-attrs drop).",
|
|
1979
1983
|
),
|
|
1984
|
+
(
|
|
1985
|
+
"fix_package_version",
|
|
1986
|
+
'rewrite package version="1.0" to "2.0" (OPF-001)',
|
|
1987
|
+
"rewrite a <package> element whose version attribute is the "
|
|
1988
|
+
"literal 1.0 -- an EPUB version epubcheck cannot parse (OPF-001 "
|
|
1989
|
+
"'error when parsing the EPUB version') -- to 2.0, the conformance "
|
|
1990
|
+
"the 2026-11-03 census's 25 carriers were all written to. Refuses "
|
|
1991
|
+
"everything else: a missing attribute (an OEB-1.0-namespace "
|
|
1992
|
+
"package needs a migration, not an attribute bump) or any other "
|
|
1993
|
+
"spelling. The gate decides whether the profile revalidation is "
|
|
1994
|
+
"a real decrease.",
|
|
1995
|
+
),
|
|
1996
|
+
(
|
|
1997
|
+
"fix_doctype",
|
|
1998
|
+
"replace a forbidden DOCTYPE with the profile's expected form (HTM-004)",
|
|
1999
|
+
"replace each content document's DOCTYPE when it carries a public "
|
|
2000
|
+
"identifier its package profile forbids: EPUB 3 requires any "
|
|
2001
|
+
"DOCTYPE to be <!DOCTYPE html> (an XHTML 1.x public doctype is a "
|
|
2002
|
+
"spec violation), and EPUB 2's OPS content docs are XHTML 1.1 (an "
|
|
2003
|
+
"XHTML 1.0 public doctype is the census's second flagged shape). "
|
|
2004
|
+
"A semantically identical declaration with different spacing is "
|
|
2005
|
+
"untouched, an ABSENT DOCTYPE is legal in both profiles and left "
|
|
2006
|
+
"alone (the first whole-library firing census measured insertion "
|
|
2007
|
+
"flat on ~2,400 of 2,545 firing books -- it never converges, "
|
|
2008
|
+
"since an equal book never applies), and a DOCTYPE carrying an "
|
|
2009
|
+
"internal subset is never touched (the subset declares entities; "
|
|
2010
|
+
"swapping it would change what the parser resolves).",
|
|
2011
|
+
),
|
|
2012
|
+
(
|
|
2013
|
+
"prune_invalid_guide_refs",
|
|
2014
|
+
"remove guide references whose target is not a content doc (OPF-032)",
|
|
2015
|
+
"remove guide <reference> elements whose href resolves to a "
|
|
2016
|
+
"manifest item that exists but is not an OPS content document "
|
|
2017
|
+
"(the 2026-11-03 census's 125-book lane: the Random House msr "
|
|
2018
|
+
"guide block pointing cover/thumbimage references at JPEGs). The "
|
|
2019
|
+
"guide is furniture, so removal is content-neutral. A reference "
|
|
2020
|
+
"to an UNDECLARED file is OPF-031 (--declare-missing-items' "
|
|
2021
|
+
"lane) and is left alone, as are external schemes and paired "
|
|
2022
|
+
"elements.",
|
|
2023
|
+
),
|
|
2024
|
+
(
|
|
2025
|
+
"fix_item_properties",
|
|
2026
|
+
"declare scripted/remote-resources on EPUB3 manifest items (OPF-014)",
|
|
2027
|
+
"scan every content document for what epubcheck wants declared "
|
|
2028
|
+
"(OPF-014 'The property X should be declared in the OPF'): a "
|
|
2029
|
+
"<script> element earns `scripted`, an absolute-scheme src, "
|
|
2030
|
+
"remote <link href>, or remote CSS url() earns "
|
|
2031
|
+
"`remote-resources` -- and merge the tokens into the owning "
|
|
2032
|
+
"manifest item's properties attribute, sorted and deduplicated. "
|
|
2033
|
+
"EPUB 3 packages only (properties is EPUB3 vocabulary; below 3 "
|
|
2034
|
+
"the attribute is itself a finding). Declares reality; changes "
|
|
2035
|
+
"no content.",
|
|
2036
|
+
),
|
|
1980
2037
|
],
|
|
1981
2038
|
"lossy strips (these delete converter-injected content; accepted on a no-worse bar)": [
|
|
1982
2039
|
(
|
|
@@ -304,6 +304,10 @@ class RepairFlags:
|
|
|
304
304
|
unterminated_attr: bool = False
|
|
305
305
|
unbound_prefix: bool = False
|
|
306
306
|
unique_identifier: bool = False
|
|
307
|
+
package_version: bool = False
|
|
308
|
+
doctype: bool = False
|
|
309
|
+
prune_guide_refs: bool = False
|
|
310
|
+
item_properties: bool = False
|
|
307
311
|
|
|
308
312
|
|
|
309
313
|
@dataclass
|
|
@@ -348,14 +352,40 @@ _PAGELIST_OPEN_RE = re.compile(r"<pageList(?=[\s/>])([^>]*)")
|
|
|
348
352
|
|
|
349
353
|
|
|
350
354
|
def fix_pagelist_class(ncx_text: str) -> tuple[str, int]:
|
|
351
|
-
"""Add ``class="pages"`` to ``<pageList>`` elements missing
|
|
355
|
+
"""Add ``id`` and ``class="pages"`` to ``<pageList>`` elements missing
|
|
356
|
+
the class attribute.
|
|
352
357
|
|
|
353
358
|
NCX 2.x requires ``class`` on ``<pageList>``; older conversions leave it
|
|
354
359
|
out and epubcheck answers RSC-005 'missing required attribute "class"'.
|
|
355
360
|
``class="pages"`` is the value upstream's own tooling emits, and a
|
|
356
361
|
pageList that already carries a class is left untouched.
|
|
362
|
+
|
|
363
|
+
The ``id`` rides along (v0.62.0): the 2026-11-03 whole-library dry run
|
|
364
|
+
measured class-only insertion flipping epubcheck into demanding ``id``
|
|
365
|
+
on the very element that just gained the class (Feel Free: bare
|
|
366
|
+
pageList validates clean, ``class="pages"`` alone draws RSC-005
|
|
367
|
+
'missing required attribute "id"', ``id`` + ``class`` validates fully
|
|
368
|
+
clean). Adding both is clean in both contexts, so the books that used
|
|
369
|
+
to regress now measure flat and the ones that used to accept still do.
|
|
370
|
+
|
|
371
|
+
The id is ``bindery-pagelist``, suffixed -2/-3/... if taken anywhere in
|
|
372
|
+
the document (the declare-missing id discipline): NCX ids must be
|
|
373
|
+
unique, and a collision would be a new finding of our own making.
|
|
357
374
|
"""
|
|
358
375
|
count = 0
|
|
376
|
+
taken: set[str] = {
|
|
377
|
+
m.group(1) for m in re.finditer(r'\bid\s*=\s*["\']([^"\']+)["\']', ncx_text)
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
def _pick_id() -> str:
|
|
381
|
+
base = "bindery-pagelist"
|
|
382
|
+
candidate = base
|
|
383
|
+
n = 2
|
|
384
|
+
while candidate in taken:
|
|
385
|
+
candidate = f"{base}-{n}"
|
|
386
|
+
n += 1
|
|
387
|
+
taken.add(candidate)
|
|
388
|
+
return candidate
|
|
359
389
|
|
|
360
390
|
def _add(m: re.Match) -> str:
|
|
361
391
|
nonlocal count
|
|
@@ -363,11 +393,291 @@ def fix_pagelist_class(ncx_text: str) -> tuple[str, int]:
|
|
|
363
393
|
if re.search(r"\bclass\s*=", attrs, re.IGNORECASE):
|
|
364
394
|
return m.group(0)
|
|
365
395
|
count += 1
|
|
366
|
-
|
|
396
|
+
if re.search(r"\bid\s*=", attrs, re.IGNORECASE):
|
|
397
|
+
return f'<pageList class="pages"{attrs}'
|
|
398
|
+
return f'<pageList id="{_pick_id()}" class="pages"{attrs}'
|
|
367
399
|
|
|
368
400
|
return _PAGELIST_OPEN_RE.sub(_add, ncx_text), count
|
|
369
401
|
|
|
370
402
|
|
|
403
|
+
# --fix-package-version: the OPF-001 lane (the 2026-11-03 triage). 25 of 26
|
|
404
|
+
# carriers carry literally `version="1.0"` (an EPUB version epubcheck cannot
|
|
405
|
+
# parse: "Version not supported"); the 26th is an OEB-1.0-namespace package
|
|
406
|
+
# with no version attribute at all (Dying Inside, the designed refusal).
|
|
407
|
+
# Only the exact literal qualifies: bumping anything else would be a guess
|
|
408
|
+
# about which conformance the producer meant. The value is what the
|
|
409
|
+
# EPUB2-targeted fixes' own package_version() reads (major 1 -> 2 keeps
|
|
410
|
+
# their licensing identical), and the gate decides whether the profile
|
|
411
|
+
# revalidation the bump unmasking-free.
|
|
412
|
+
_PACKAGE_VERSION_ONE_RE = re.compile(
|
|
413
|
+
r"""(<(?:[\w.-]+:)?package\b[^>]*?\bversion\s*=\s*)(["'])1\.0\2""",
|
|
414
|
+
re.IGNORECASE,
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def fix_package_version(opf_text: str) -> tuple[str, int]:
|
|
419
|
+
"""Rewrite ``version="1.0"`` on the ``<package>`` element to ``"2.0"``.
|
|
420
|
+
|
|
421
|
+
An unparseable EPUB version is OPF-001 and the package falls out of
|
|
422
|
+
every version-aware check. Only the exact literal ``1.0`` (double or
|
|
423
|
+
single quoted) qualifies: a missing attribute, ``1.1``, or any other
|
|
424
|
+
spelling refuses the whole document. Everything else about the OPF is
|
|
425
|
+
untouched; the gate measures whether the 2.0 profile revalidation the
|
|
426
|
+
bump buys is a real decrease.
|
|
427
|
+
"""
|
|
428
|
+
out, n = _PACKAGE_VERSION_ONE_RE.subn(r"\g<1>\g<2>2.0\g<2>", opf_text)
|
|
429
|
+
return out, n
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
# --fix-doctype: the HTM-004 lane (the 2026-11-03 triage, 135 books in two
|
|
433
|
+
# shapes: EPUB3 packages carrying XHTML 1.1 doctypes, EPUB2 packages carrying
|
|
434
|
+
# XHTML 1.0 or none). The expected form is package-profile-driven, matching
|
|
435
|
+
# epubcheck's own expectation strings verbatim.
|
|
436
|
+
_XHTML11_DOCTYPE = (
|
|
437
|
+
'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" '
|
|
438
|
+
'"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">'
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _scan_doctype_span(text: str) -> tuple[int, int] | None:
|
|
443
|
+
"""The exact (start, end) span of a DOCTYPE declaration, honouring the
|
|
444
|
+
bracketed internal subset: a naive `[^>]*` stops at the first `>`, which
|
|
445
|
+
a subset's conditional sections can carry. None when unterminated."""
|
|
446
|
+
m = re.search(r"<!DOCTYPE", text, re.IGNORECASE)
|
|
447
|
+
if m is None:
|
|
448
|
+
return None
|
|
449
|
+
depth = 0
|
|
450
|
+
i = m.end()
|
|
451
|
+
while i < len(text):
|
|
452
|
+
ch = text[i]
|
|
453
|
+
if ch == "[":
|
|
454
|
+
depth += 1
|
|
455
|
+
elif ch == "]":
|
|
456
|
+
depth -= 1
|
|
457
|
+
elif ch == ">" and depth <= 0:
|
|
458
|
+
return m.start(), i + 1
|
|
459
|
+
i += 1
|
|
460
|
+
return None
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def fix_doctype(text: str, major: int | None) -> tuple[str, int]:
|
|
464
|
+
"""Replace one content document's DOCTYPE when it carries a public
|
|
465
|
+
identifier its package profile forbids (the HTM-004 repair).
|
|
466
|
+
|
|
467
|
+
EPUB 3 requires any DOCTYPE to be ``<!DOCTYPE html>`` -- an XHTML 1.x
|
|
468
|
+
public doctype on an EPUB3 document is a spec violation and an oracle
|
|
469
|
+
finding. EPUB 2's OPS content docs are XHTML 1.1; an XHTML 1.0 public
|
|
470
|
+
doctype there is the census's second flagged shape (Hound of the
|
|
471
|
+
Baskervilles: found 1.0 Strict, expected XHTML 1.1). Whitespace-
|
|
472
|
+
normalized equality decides no-op, and an internal subset is left
|
|
473
|
+
alone (the subset declares entities; replacing it changes what the
|
|
474
|
+
parser resolves). An unknown package version licenses nothing.
|
|
475
|
+
|
|
476
|
+
An ABSENT doctype is deliberately not filled in (v0.62.0 tightening,
|
|
477
|
+
after the first whole-library firing census): a missing DOCTYPE is
|
|
478
|
+
legal in both profiles and epubcheck tolerates it -- the census's
|
|
479
|
+
found-"" carriers are a minority the oracle flags for its own reasons
|
|
480
|
+
no byte predicate can predict -- so insertion measured flat on ~2,400
|
|
481
|
+
of the 2,545 books the first firing swept, never converged (an equal
|
|
482
|
+
book never applies, so every future sweep refired), and risked
|
|
483
|
+
unmasking by licensing the strict DTD on docs that parsed loosely.
|
|
484
|
+
Fires only where the declaration exists and is wrong.
|
|
485
|
+
"""
|
|
486
|
+
if major is None:
|
|
487
|
+
return text, 0
|
|
488
|
+
expected = "<!DOCTYPE html>" if major >= 3 else _XHTML11_DOCTYPE
|
|
489
|
+
span = _scan_doctype_span(text)
|
|
490
|
+
if span is None:
|
|
491
|
+
# absent: legal in both profiles; see the docstring
|
|
492
|
+
return text, 0
|
|
493
|
+
start, end = span
|
|
494
|
+
existing = text[start:end]
|
|
495
|
+
if " ".join(existing.split()) == " ".join(expected.split()):
|
|
496
|
+
return text, 0
|
|
497
|
+
if "[" in existing:
|
|
498
|
+
# an internal subset declares entities: swapping the doctype would
|
|
499
|
+
# silently change what the parser resolves (fix-cdata-terminator's
|
|
500
|
+
# neighbour). Leave measured.
|
|
501
|
+
return text, 0
|
|
502
|
+
return f"{text[:start]}{expected}{text[end:]}", 1
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
# --prune-invalid-guide-refs: the OPF-032 lane (the 2026-11-03 triage, 125
|
|
506
|
+
# books; one producer fingerprint dominates: the Random House `msr` guide
|
|
507
|
+
# block pointing cover/thumbimage references at JPEGs). A guide reference
|
|
508
|
+
# is valid only when its target resolves to a manifest item of OPS content
|
|
509
|
+
# doc type; anything else epubcheck answers OPF-032. The guide is furniture
|
|
510
|
+
# (never rendered content), so removal is content-neutral.
|
|
511
|
+
_CONTENT_MEDIA_TYPES = frozenset(
|
|
512
|
+
{
|
|
513
|
+
"application/xhtml+xml",
|
|
514
|
+
"text/html",
|
|
515
|
+
"application/x-dtbook+xml",
|
|
516
|
+
}
|
|
517
|
+
)
|
|
518
|
+
_GUIDE_REF_ELEMENT_RE = re.compile(
|
|
519
|
+
r"""<reference\b[^>]*?/>|<reference\b[^>]*>.*?</reference>""",
|
|
520
|
+
re.IGNORECASE | re.DOTALL,
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def _manifest_media_type_map(opf_text: str, opf_dir: str) -> dict[str, str]:
|
|
525
|
+
"""resolved archive path -> declared media type, for the guide prune."""
|
|
526
|
+
out: dict[str, str] = {}
|
|
527
|
+
for m in _ITEM_TAG_RE.finditer(opf_text):
|
|
528
|
+
tag = m.group(0)
|
|
529
|
+
hm = _HREF_ATTR_RE.search(tag)
|
|
530
|
+
if not hm:
|
|
531
|
+
continue
|
|
532
|
+
resolved = _resolve_href(opf_dir, hm.group(2))
|
|
533
|
+
if resolved is None:
|
|
534
|
+
continue
|
|
535
|
+
tm = re.search(
|
|
536
|
+
r"""\bmedia-type\s*=\s*(["'])((?:(?!\1).)*)\1""", tag, re.IGNORECASE
|
|
537
|
+
)
|
|
538
|
+
if tm is not None:
|
|
539
|
+
out[resolved] = tm.group(2)
|
|
540
|
+
return out
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def prune_invalid_guide_refs(opf_text: str, opf_dir: str) -> tuple[str, int]:
|
|
544
|
+
"""Remove guide ``<reference>`` elements whose target is declared but is
|
|
545
|
+
not an OPS content document (OPF-032).
|
|
546
|
+
|
|
547
|
+
The predicate is deliberate: a reference to an UNDECLARED file is
|
|
548
|
+
OPF-031, declare-missing's lane; an external scheme is nobody's guide;
|
|
549
|
+
a self-closed element is the only shape removed (a paired reference
|
|
550
|
+
with children is unheard of in the census, and a botched span would
|
|
551
|
+
eat the guide). Nothing else about the guide is rewritten.
|
|
552
|
+
"""
|
|
553
|
+
media_types = _manifest_media_type_map(opf_text, opf_dir)
|
|
554
|
+
count = 0
|
|
555
|
+
|
|
556
|
+
def repl(m: re.Match) -> str:
|
|
557
|
+
nonlocal count
|
|
558
|
+
tag = m.group(0)
|
|
559
|
+
hm = _HREF_ATTR_RE.search(tag)
|
|
560
|
+
if hm is None:
|
|
561
|
+
return tag
|
|
562
|
+
resolved = _resolve_href(opf_dir, hm.group(2))
|
|
563
|
+
if resolved is None:
|
|
564
|
+
return tag
|
|
565
|
+
declared_type = media_types.get(resolved)
|
|
566
|
+
if declared_type is None or declared_type in _CONTENT_MEDIA_TYPES:
|
|
567
|
+
return tag
|
|
568
|
+
count += 1
|
|
569
|
+
return ""
|
|
570
|
+
|
|
571
|
+
return _GUIDE_REF_ELEMENT_RE.sub(repl, opf_text), count
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
# --fix-item-properties: the OPF-014 lane (the 2026-11-03 triage, 74 books).
|
|
575
|
+
# EPUB3 manifest items must declare what their documents contain: `scripted`
|
|
576
|
+
# when the document carries scripting (a <script> element, an on* handler,
|
|
577
|
+
# or a <form> — epubcheck 5.3 demands scripted for form documents, verified
|
|
578
|
+
# on the Rustonomicon's scraped GitHub page where tag-off bisection isolated
|
|
579
|
+
# the form), `svg` when the document carries an <svg>, and
|
|
580
|
+
# `remote-resources` when it loads resources from outside the container
|
|
581
|
+
# (src=/href= attributes and CSS url() values with an absolute scheme).
|
|
582
|
+
# Declaring reality changes no content.
|
|
583
|
+
_SCRIPT_TAG_RE = re.compile(r"<script\b", re.IGNORECASE)
|
|
584
|
+
_FORM_TAG_RE = re.compile(r"<form\b", re.IGNORECASE)
|
|
585
|
+
_HANDLER_ATTR_RE = re.compile(r"""\son[a-z]+\s*=\s*["']""", re.IGNORECASE)
|
|
586
|
+
_SVG_TAG_RE = re.compile(r"<svg\b", re.IGNORECASE)
|
|
587
|
+
_REMOTE_SRC_RE = re.compile(r"""\ssrc\s*=\s*(["'])https?://""", re.IGNORECASE)
|
|
588
|
+
_REMOTE_LINK_HREF_RE = re.compile(
|
|
589
|
+
r"""<link\b[^>]*?\shref\s*=\s*(["'])https?://""", re.IGNORECASE
|
|
590
|
+
)
|
|
591
|
+
_REMOTE_CSS_URL_RE = re.compile(r"""url\(\s*["']?https?://""", re.IGNORECASE)
|
|
592
|
+
_ITEM_PROPERTIES_RE = re.compile(
|
|
593
|
+
r"""((?:^|\s)properties\s*=\s*)(["'])((?:(?!\2).)*)(\2)""", re.IGNORECASE
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def collect_item_properties(zin: zipfile.ZipFile) -> dict[str, frozenset[str]]:
|
|
598
|
+
"""The item-properties census: resolved content-doc path -> the EPUB3
|
|
599
|
+
property tokens the document's own bytes demand. `scripted` from a
|
|
600
|
+
`<script>` element, an on* handler attribute, or a `<form>`; `svg` from
|
|
601
|
+
an `<svg>` element; `remote-resources` from an absolute-scheme src
|
|
602
|
+
attribute, a remote `<link href>`, or a remote CSS url(). Non-UTF-8
|
|
603
|
+
documents are skipped (their contents are nobody's guess)."""
|
|
604
|
+
props: dict[str, set[str]] = {}
|
|
605
|
+
for item in zin.infolist():
|
|
606
|
+
low = item.filename.lower()
|
|
607
|
+
if not low.endswith(CONTENT_SUFFIXES):
|
|
608
|
+
continue
|
|
609
|
+
try:
|
|
610
|
+
text = zin.read(item).decode("utf-8")
|
|
611
|
+
except (UnicodeDecodeError, RuntimeError):
|
|
612
|
+
continue
|
|
613
|
+
found: set[str] = set()
|
|
614
|
+
if (
|
|
615
|
+
_SCRIPT_TAG_RE.search(text)
|
|
616
|
+
or _FORM_TAG_RE.search(text)
|
|
617
|
+
or _HANDLER_ATTR_RE.search(text)
|
|
618
|
+
):
|
|
619
|
+
found.add("scripted")
|
|
620
|
+
if _SVG_TAG_RE.search(text):
|
|
621
|
+
found.add("svg")
|
|
622
|
+
if (
|
|
623
|
+
_REMOTE_SRC_RE.search(text)
|
|
624
|
+
or _REMOTE_LINK_HREF_RE.search(text)
|
|
625
|
+
or _REMOTE_CSS_URL_RE.search(text)
|
|
626
|
+
):
|
|
627
|
+
found.add("remote-resources")
|
|
628
|
+
if not found:
|
|
629
|
+
continue
|
|
630
|
+
# key in the same spelling _resolve_href produces for a manifest
|
|
631
|
+
# item's href (normpath of the archive path), so the patcher's
|
|
632
|
+
# lookup matches
|
|
633
|
+
props[posixpath.normpath(item.filename)] = frozenset(found)
|
|
634
|
+
return props
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def fix_manifest_item_properties(
|
|
638
|
+
opf_text: str, opf_dir: str, props_map: dict[str, frozenset[str]]
|
|
639
|
+
) -> tuple[str, int]:
|
|
640
|
+
"""Merge the census tokens into the owning manifest items' `properties`
|
|
641
|
+
attributes (EPUB3 packages only; the caller gates). Tokens merge sorted
|
|
642
|
+
and deduplicated, so `nav` + `scripted` reads `nav scripted`; an item
|
|
643
|
+
already carrying every token is untouched, and a doc with no manifest
|
|
644
|
+
item stays untouched (declaring an undeclared file is declare-missing's
|
|
645
|
+
lane). Returns (text, items-changed)."""
|
|
646
|
+
if not props_map:
|
|
647
|
+
return opf_text, 0
|
|
648
|
+
count = 0
|
|
649
|
+
|
|
650
|
+
def repl(m: re.Match) -> str:
|
|
651
|
+
nonlocal count
|
|
652
|
+
tag = m.group(0)
|
|
653
|
+
hm = _HREF_ATTR_RE.search(tag)
|
|
654
|
+
if hm is None:
|
|
655
|
+
return tag
|
|
656
|
+
resolved = _resolve_href(opf_dir, hm.group(2))
|
|
657
|
+
if resolved is None:
|
|
658
|
+
return tag
|
|
659
|
+
needed = props_map.get(resolved)
|
|
660
|
+
if not needed:
|
|
661
|
+
return tag
|
|
662
|
+
pm = _ITEM_PROPERTIES_RE.search(tag)
|
|
663
|
+
if pm is None:
|
|
664
|
+
merged = " ".join(sorted(needed))
|
|
665
|
+
count += 1
|
|
666
|
+
return (
|
|
667
|
+
tag[: tag.rindex(">")]
|
|
668
|
+
+ f' properties="{merged}"'
|
|
669
|
+
+ tag[tag.rindex(">") :]
|
|
670
|
+
)
|
|
671
|
+
existing = set(pm.group(3).split())
|
|
672
|
+
if existing >= needed:
|
|
673
|
+
return tag
|
|
674
|
+
merged = " ".join(sorted(existing | needed))
|
|
675
|
+
count += 1
|
|
676
|
+
return tag[: pm.start(3)] + merged + tag[pm.end(3) :]
|
|
677
|
+
|
|
678
|
+
return _ITEM_TAG_RE.sub(repl, opf_text), count
|
|
679
|
+
|
|
680
|
+
|
|
371
681
|
# The fixed EPUB3-semantics set (issue #11): `role` and `aria-hidden` join
|
|
372
682
|
# the original three. Deliberately NOT here: `hidden` (stripping it would
|
|
373
683
|
# REVEAL content the author hid, a rendering change) and `epub:prefix`
|
|
@@ -2248,6 +2558,9 @@ def repair_epub(
|
|
|
2248
2558
|
undeclared: frozenset[str] = frozenset()
|
|
2249
2559
|
if flags.declare_missing and opf_text is not None:
|
|
2250
2560
|
undeclared = collect_undeclared_resources(zin, opf_text, opf_dir, present)
|
|
2561
|
+
item_props: dict[str, frozenset[str]] = {}
|
|
2562
|
+
if flags.item_properties:
|
|
2563
|
+
item_props = collect_item_properties(zin)
|
|
2251
2564
|
|
|
2252
2565
|
# --strip-stub-docs: the whole-book stub identity decision needs the
|
|
2253
2566
|
# spine order and every spine doc's visible text, so it runs once up
|
|
@@ -2532,8 +2845,19 @@ def repair_epub(
|
|
|
2532
2845
|
or flags.strip_anchors
|
|
2533
2846
|
or flags.unbound_prefix
|
|
2534
2847
|
or flags.unique_identifier
|
|
2848
|
+
or flags.package_version
|
|
2849
|
+
or flags.prune_guide_refs
|
|
2850
|
+
or flags.item_properties
|
|
2535
2851
|
):
|
|
2536
2852
|
opf_changed = False
|
|
2853
|
+
if flags.package_version:
|
|
2854
|
+
# First in the branch: the version declaration is the
|
|
2855
|
+
# package's self-description, and every later reader of
|
|
2856
|
+
# this text should see the repaired spelling.
|
|
2857
|
+
text, n = fix_package_version(text)
|
|
2858
|
+
if n:
|
|
2859
|
+
report.add({"package_version_fixed": n})
|
|
2860
|
+
opf_changed = True
|
|
2537
2861
|
if flags.unbound_prefix:
|
|
2538
2862
|
# First in the branch: the unbound prefix makes the whole
|
|
2539
2863
|
# package unparseable, so the parseability scar comes off
|
|
@@ -2585,6 +2909,23 @@ def repair_epub(
|
|
|
2585
2909
|
if n:
|
|
2586
2910
|
report.add({"manifest_items_declared": n})
|
|
2587
2911
|
opf_changed = True
|
|
2912
|
+
if flags.prune_guide_refs:
|
|
2913
|
+
text, n = prune_invalid_guide_refs(text, opf_dir)
|
|
2914
|
+
if n:
|
|
2915
|
+
report.add({"invalid_guide_refs_pruned": n})
|
|
2916
|
+
opf_changed = True
|
|
2917
|
+
if flags.item_properties:
|
|
2918
|
+
# EPUB3 vocabulary: `properties` on a manifest item is
|
|
2919
|
+
# illegal below EPUB 3 (the same profile line the
|
|
2920
|
+
# strip-epub3-attrs gate draws), so the census result is
|
|
2921
|
+
# consumed only where it can be declared honestly.
|
|
2922
|
+
if pkg_major is not None and pkg_major >= 3:
|
|
2923
|
+
text, n = fix_manifest_item_properties(
|
|
2924
|
+
text, opf_dir, item_props
|
|
2925
|
+
)
|
|
2926
|
+
if n:
|
|
2927
|
+
report.add({"item_properties_added": n})
|
|
2928
|
+
opf_changed = True
|
|
2588
2929
|
if flags.strip_stub_docs and stub_hrefs:
|
|
2589
2930
|
text, n_items, n_refs, stub_ids = strip_stub_manifest(
|
|
2590
2931
|
text, opf_dir, stub_hrefs
|
|
@@ -2733,6 +3074,10 @@ def repair_epub(
|
|
|
2733
3074
|
text, n = fix_missing_title(text)
|
|
2734
3075
|
if n:
|
|
2735
3076
|
counts["fix_missing_title"] = n
|
|
3077
|
+
if flags.doctype:
|
|
3078
|
+
text, n = fix_doctype(text, pkg_major)
|
|
3079
|
+
if n:
|
|
3080
|
+
counts["doctype_normalized"] = n
|
|
2736
3081
|
if flags.id_colons:
|
|
2737
3082
|
text, n = fix_id_colons(text)
|
|
2738
3083
|
if n:
|