bindery-cli 0.53.1__tar.gz → 0.54.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/AGENTS.md +1 -1
  2. bindery_cli-0.54.0/OBSERVE-RESEARCH.md +541 -0
  3. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/PKG-INFO +1 -1
  4. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/patchnotes.md +52 -0
  5. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/pyproject.toml +1 -1
  6. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/roadmap.md +1 -0
  7. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/spec.md +17 -4
  8. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/__init__.py +1 -1
  9. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/transforms.py +128 -28
  10. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_transforms.py +107 -1
  11. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/uv.lock +1 -1
  12. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/.github/FUNDING.yml +0 -0
  13. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/.github/workflows/ci.yml +0 -0
  14. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/.github/workflows/publish.yml +0 -0
  15. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/.gitignore +0 -0
  16. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/LICENSE +0 -0
  17. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/README.md +0 -0
  18. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/docs/screenshots/library-sweep.png +0 -0
  19. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/logo.svg +0 -0
  20. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/plugin/__init__.py +0 -0
  21. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/project.done +0 -0
  22. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/run_tests.sh +0 -0
  23. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/scripts/FastSweep.java +0 -0
  24. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/scripts/build_plugin.py +0 -0
  25. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/scripts/fast_sweep.py +0 -0
  26. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/__main__.py +0 -0
  27. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/audit.py +0 -0
  28. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/cli.py +0 -0
  29. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/epub.py +0 -0
  30. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/library.py +0 -0
  31. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/observe.py +0 -0
  32. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/pagination.py +0 -0
  33. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/reserialize.py +0 -0
  34. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/validate.py +0 -0
  35. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/src/bindery/watermark.py +0 -0
  36. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_audit.py +0 -0
  37. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_cli.py +0 -0
  38. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_comment_hyphens.py +0 -0
  39. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_contamination.py +0 -0
  40. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_cover_tocdrift.py +0 -0
  41. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_epub.py +0 -0
  42. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_flags_real_core.py +0 -0
  43. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_flags_wiring.py +0 -0
  44. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_help.py +0 -0
  45. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_library.py +0 -0
  46. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_matcher_hardening.py +0 -0
  47. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_observe.py +0 -0
  48. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_pagination.py +0 -0
  49. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_plugin.py +0 -0
  50. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_reserialize.py +0 -0
  51. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_space_renames.py +0 -0
  52. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_spine_guard.py +0 -0
  53. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_validate.py +0 -0
  54. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_version.py +0 -0
  55. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_watermark.py +0 -0
  56. {bindery_cli-0.53.1 → bindery_cli-0.54.0}/tests/test_xml_hardening.py +0 -0
@@ -132,7 +132,7 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
132
132
 
133
133
  ## Layout
134
134
 
135
- - `src/bindery/transforms.py`: pure `str -> (str, int)` text transforms (including `strip_broken_tags`). Since v0.32.0: `strip_attrs_in_start_tags` anchors attribute edits on the quote-aware start-tag matcher, `_outside_protected` forwards arguments (context-carrying transforms can be decorated), `fix_id_colons` matches only the bare `id` attribute and only internal fragments (`fix_ncx_src_fragments` carries renames into the NCX), `fix_ncx_playorder` is anchored to `<navPoint>` start tags, and the css selector boundary covers namespaced and functional selector forms. HAZARD (the 0.41.0 spin): the quote-aware tag-prefix alternation (double-quoted | single-quoted | [^>]) OVERLAPS ([^>] matches quotes too), so a plain `*` re-partitions exponentially on failing candidates; every such greedy loop must carry the possessive `*+` (re 3.11+, the plugin's minimum Calibre). Since v0.42.0 the census is complete and pinned by tests/test_matcher_hardening.py: every greedy loop of this family is possessive (the three 0.41.1 rewrites plus _START_TAG_RE, _COVER_META_RE, _GUIDE_REF_RE, prune_dangling_edges' spine|item|itemref matcher, _ITEM_TAG_RE, _SPINE_ITEM_RE; possessive is byte-identical on success because the loop is always followed by a literal `>` the [^>] branch cannot consume), and the five LAZY siblings (_VOID_RE, the transforms <img> matcher, and epub's link/a/img matchers) use the disjoint catch-all `[^>"']` instead: mutually exclusive branches cannot re-partition, and an unterminated quote inside a tag now fails to match rather than pairing across markup. NEW tag matchers: quote-aware possessive for greedy loops, the disjoint catch-all for lazy ones, and the tempered-quote idiom (["'])((?:(?!\1).)*)\1 for attribute values. Since v0.37.0: `fix_ncx_playorder` strips quotes before its already-correct comparison (regex groups carry quote characters; comparing quoted values against bare numbers counted phantom fixes on every sequential NCX and rewrote single-quoted correct attributes to double quotes).
135
+ - `src/bindery/transforms.py`: pure `str -> (str, int)` text transforms (including `strip_broken_tags`). Since v0.32.0: `strip_attrs_in_start_tags` anchors attribute edits on the quote-aware start-tag matcher, `_outside_protected` forwards arguments (context-carrying transforms can be decorated), `fix_id_colons` matches only the bare `id` attribute and only internal fragments (`fix_ncx_src_fragments` carries renames into the NCX), `fix_ncx_playorder` is anchored to `<navPoint>` start tags, and the css selector boundary covers namespaced and functional selector forms. HAZARD (the 0.41.0 spin): the quote-aware tag-prefix alternation (double-quoted | single-quoted | [^>]) OVERLAPS ([^>] matches quotes too), so a plain `*` re-partitions exponentially on failing candidates; every such greedy loop must carry the possessive `*+` (re 3.11+, the plugin's minimum Calibre). Since v0.42.0 the census is complete and pinned by tests/test_matcher_hardening.py: every greedy loop of this family is possessive (the three 0.41.1 rewrites plus _START_TAG_RE, _COVER_META_RE, _GUIDE_REF_RE, prune_dangling_edges' spine|item|itemref matcher, _ITEM_TAG_RE, _SPINE_ITEM_RE; possessive is byte-identical on success because the loop is always followed by a literal `>` the [^>] branch cannot consume), and the five LAZY siblings (_VOID_RE, the transforms <img> matcher, and epub's link/a/img matchers) use the disjoint catch-all `[^>"']` instead: mutually exclusive branches cannot re-partition, and an unterminated quote inside a tag now fails to match rather than pairing across markup. NEW tag matchers: quote-aware possessive for greedy loops, the disjoint catch-all for lazy ones, and the tempered-quote idiom (["'])((?:(?!\1).)*)\1 for attribute values. Since v0.37.0: `fix_ncx_playorder` strips quotes before its already-correct comparison (regex groups carry quote characters; comparing quoted values against bare numbers counted phantom fixes on every sequential NCX and rewrote single-quoted correct attributes to double quotes). Since v0.54.0 (the 2026-10-24 surgery): `fix_ncx_playorder` is CONTRACT-GATED -- it tokenizes navPoint/navTarget/pageTarget start tags plus their `<content src>` targets, returns unchanged when the NCX already satisfies epubcheck's playOrder contract (same target shares one value, distinct targets distinct values, values contiguous 1..K), and re-sequences by first-appearance target order only on a violation; the old unconditional 1..N renumbering rewrote valid TOCs into violations (211 of 579 firings gate-rejected, 17 real fixes, 347 no-ops). The same fix extends `_PROTECTED_RE` with the Calibre `/*<![CDATA[*/ ... /*]]>*/` wrapper as one protected unit (the bare CDATA alternative ends inside the closing `/*]]>`, which exposed the trailing `*/` to strip_prolog_junk's amputation: 111 phantom-fix rejections).
136
136
  - `src/bindery/pagination.py`: the opt-in lossy page-number strip (runhead detection, page-layer decision, block-centric removal/merge, safety nets).
137
137
  - `src/bindery/watermark.py`: the opt-in lossy watermark strip (anchored and anchorless signature removal).
138
138
  - `src/bindery/reserialize.py`: structural repair via `html5lib`.
@@ -0,0 +1,541 @@
1
+ # Observe research: the error-code surgery (2026-10-24)
2
+
3
+ Working research file behind bindery observe's first library census
4
+ (`~/docs/Calibre Library.backups/observe/observe-2026-10-24-base.observe`).
5
+ Question: what exactly is triggering the library's four dominant epubcheck
6
+ code classes, and what exactly makes the gate reject the 313 repairs it
7
+ refused on the whole-library dry run? Specimen books live in
8
+ `testing_facility/error-surgery-2026-10-24/` (gitignored; never modified
9
+ copies of library books). This file is the living record; findings land here
10
+ as the investigation proceeds.
11
+
12
+ ## The oracle's own definitions (epubcheck 5.3.0 MessageBundle.properties)
13
+
14
+ | Code | epubcheck's message | Library mass |
15
+ |---|---|---|
16
+ | RSC-005 | `Error while parsing file: %1$s` (the generic XML parse-error class; a zoo of distinct underlying defects) | 17,105 occurrences in 3,323 books |
17
+ | RSC-007 | `Referenced resource "%1$s" could not be found in the EPUB.` | 14,341 in 692 books (~20.7/book) |
18
+ | RSC-026 | `URL "%1$s" leaks outside the container (it is not a valid-relative-ocf-URL-with-fragment string)` | 12,607 in only 18 books (~700/book) |
19
+ | RSC-020 | `"%1$s" is not a valid URL (%2$s)` | 10,102 in 353 books |
20
+ | PKG-010 | `The file name "%1$s" contains spaces, which may create interoperability issues with older reading systems.` | 8,294 in 287 books (~29/book) |
21
+
22
+ Siblings in the top ten: RSC-001 (`File "%1$s" could not be found`, 1,546 in
23
+ 39 books), HTM-004 (irregular DOCTYPE, in the Wandering Inn signature),
24
+ RSC-008 (referenced resource not declared in the OPF manifest), CSS-007,
25
+ RSC-004 (encrypted files), OPF-029/030.
26
+
27
+ ## The structural fact that frames everything
28
+
29
+ The whole-library preview ran the DEFAULT flag set (the five always-on
30
+ well-formedness fixes plus the NCX pipeline and mimetype repair). Every fix
31
+ that addresses the four classes above is opt-in:
32
+
33
+ - RSC-007 -> `--prune-missing-resources` (drops references to absent files)
34
+ - PKG-010 -> `--encode-url-spaces` (renames the entries and rewrites every
35
+ reference: OPF, NCX, content, CSS url())
36
+ - RSC-005 families -> spread across `--fix-ids`, `--strip-bad-attrs`,
37
+ `--strip-epub3-attrs`, `--fix-misnested-inline`, `--fix-id-colons`,
38
+ `--fix-unterminated-attr`, `--reserialize`, and friends
39
+ - RSC-026 / RSC-020 -> no dedicated repair today (candidate analysis below)
40
+
41
+ So the 3,156 `nochange` books are mostly books whose defects belong to
42
+ opt-in classes the default pass never attempts. That is by design, not a
43
+ failure. The 313 `reject`s are the interesting anomaly: there the DEFAULT
44
+ fixes fired and the gate refused because the error count went UP.
45
+
46
+ ## The 313 rejects: first shape
47
+
48
+ From the 2026-10-24 dry run (`bindery-dryrun-2026-10-24.json`):
49
+
50
+ - fix mix: `fix_ncx_playorder`-only 199, `strip_prolog_junk`-only 73,
51
+ `fix_named_entities`+`strip_prolog_junk` 29, mixes 12
52
+ - delta shapes: overwhelmingly +1 or +2 errors ((2,4) x50, (1,2) x42,
53
+ (2,3) x35, (5,6) x29, (4,5) x25, (3,4) x19)
54
+
55
+ Working hypothesis (to be proven or killed by the message-diff surgery):
56
+ the repair un-masks latent errors. epubcheck's parser stops counting at a
57
+ malformed construct; when the default pass removes junk before the XML
58
+ prolog, normalizes entities, or renumbers the NCX, validation reaches
59
+ deeper into the file and surfaces errors that were always there. The gate
60
+ sees a count increase on a no-fatal book and refuses, which is correct
61
+ behavior for a `strict error decrease` bar, but the books' TRUE defect set
62
+ is unchanged. The question the message diffs answer: does the delta
63
+ consist of NEW message families (un-masking) or more of the same message
64
+ (the fix genuinely breaking something)?
65
+
66
+ ## Surgery log
67
+
68
+ ### RSC-026: SOLVED. The `/root/work/img` equation scar (wave 1, agent B)
69
+
70
+ Specimen: the three Feynman Lectures volumes (facility copies,
71
+ `testing_facility/error-surgery-2026-10-24/rsc026/`). On vol3, **100% of
72
+ RSC-026 occurrences (10,044) are `<img src="/root/work/img/<32-hex>.png">`**
73
+ tags sitting mid-sentence where inline equations render. Zero absolute
74
+ http(s) used internally, zero `file://`, zero `../` escapes, zero
75
+ percent-encoding.
76
+
77
+ Mechanism: an upstream conversion pipeline ran as root with a scratch
78
+ workdir `/root/work`, rendered every equation and figure to a PNG named by
79
+ its md5, and emitted root-absolute `src` references. On the build machine
80
+ those paths resolved, so the conversion "succeeded" and the HTML was zipped
81
+ without any packager ever rewriting or embedding the images (Calibre embeds
82
+ only images it can find relative to a source tree it knows about;
83
+ root-absolute refs into its own temp dir are invisible once that dir is
84
+ gone). `class="calibre10"` fingerprints the Calibre pass-through. The
85
+ 3,499 distinct URLs are uniformly-random md5s (3,414 distinct 4-hex
86
+ prefixes), i.e. content hashes, not sequence numbers; every one of the 21
87
+ content docs is affected (top: 902, 801, 791 occurrences per file).
88
+
89
+ Two compounding facts:
90
+
91
+ 1. **RSC-026 and RSC-007 are 1:1 twins for the same defect** on this book:
92
+ the URL is invalid (leaks the container) AND the resource is missing.
93
+ On vol3: RSC-026 10,044 text occurrences / RSC-007 10,106, of which
94
+ 10,044 are the same equation images. Together they are 99.2% of vol3's
95
+ error mass. The remaining 62 RSC-007s are a separate small habit:
96
+ hand-written cross-volume chapter links (`I_37.html`, `II_34.html`) that
97
+ assume the other volumes' files live in the same package.
98
+ 2. **No repair in bindery's inventory should touch this.**
99
+ `--prune-missing-resources` would delete the references, which means
100
+ deleting every equation in a physics text: content destruction, not
101
+ repair. These books need re-sourcing (the census says so; the flag
102
+ inventory has nothing honest to offer). The 18-book RSC-026 population
103
+ (12,607 aggregated messages) is almost certainly all the same producer
104
+ fingerprint (root-absolute md5 workdir images); confirm from the books
105
+ array when the v0.53.1 baseline lands.
106
+
107
+ #### RSC-026 population revised (wave 2): the 18 carriers are NOT one producer
108
+
109
+ The fingerprint check across 6 of the 15 non-Feynman carriers (Dawn of
110
+ Everything, Embracing Defeat, Feet of Clay, all three Gotrek & Felix
111
+ omnibuses; specimens in the facility) kills the one-fingerprint
112
+ generalization: zero `/root/work` literals anywhere outside Feynman.
113
+ RSC-026 is epubcheck's generic "URL escapes the container" code, so it
114
+ aggregates unrelated defects:
115
+
116
+ - **Pipeline A (Feynman x3)**: the root-absolute md5 equation images. This
117
+ is the entire mass (12,607 aggregated messages library-wide is
118
+ essentially all Feynman).
119
+ - **Pipeline B (Calibre flat EPUB2 repacks)**: HTML flattened to zip root
120
+ while refs kept pre-flatten `../` prefixes. Three sub-shapes: cover img
121
+ `../Images/0000NN.jpg` (G&F family, byte-identical template, plus a case
122
+ mismatch: the dir is lowercase `images/`), content img `../images/...`
123
+ (Dawn of Everything), nav anchor `../Text/...#fileposNNN` (Feet of Clay).
124
+ Tiny counts (1-9 per book). Notably REPAIRABLE in principle: the G&F and
125
+ Dawn images exist in the archive under their flat names, so a re-point
126
+ (basename exists elsewhere in the archive) would fix them; that is a
127
+ candidate repair class, not an existing flag.
128
+ - **Pipeline C (Embracing Defeat)**: 9 garbage hrefs `../../../../l%20` —
129
+ broken source-conversion cross-references; strip-broken-anchors class.
130
+
131
+ Also surfaced: Dawn of Everything carries ~13,490 RSC-005 text
132
+ occurrences, another heavyweight outside agent A's sample; the stratified
133
+ resample matters.
134
+
135
+ ### MEASUREMENT FINDING: the census counts aggregated messages, not text-summary occurrences
136
+
137
+ Direct run on vol3: the epubcheck **text summary** says `20,307 errors`;
138
+ the epubcheck **`--json`** (and identically `CheckingReport.generate()`,
139
+ which the census harness and the gate both read) says **`nError: 7030`**.
140
+ Both are real; they are different units:
141
+
142
+ - JSON: one **message entry** per distinct defect (ID + message text),
143
+ carrying a `locations` array with every occurrence site (plus
144
+ `additionalLocations`). vol3: RSC-007 = 3,528 entries, RSC-026 = 3,499
145
+ entries (exactly the 3,499 distinct equation URLs), RSC-005 = 3 entries
146
+ (the `valign` family), total 7,030.
147
+ - Text summary: sums the **locations** (20,307 = every site individually).
148
+
149
+ Consequences:
150
+
151
+ 1. The gate and the whole observe stack are internally consistent: before,
152
+ after, the dry run, the preview, and the census all read
153
+ `checker.nError`, the same aggregated scale. No accept/reject verdict
154
+ changes.
155
+ 2. The `.observe` artifact's `codes.occurrences` field is **mislabeled**:
156
+ it counts aggregated message entries (distinct defect texts), not
157
+ occurrences. Comparing it to a manual `epubcheck` text run will
158
+ understate by the aggregation factor (vol3: 7,030 vs 20,307, 2.9x).
159
+ The field should be renamed (`codes.messages`) and/or paired with a
160
+ true occurrences count (sum of `locations` lengths), at the next schema
161
+ revision. Until then: every number in this report and in the census
162
+ that came from the JSON path is in aggregated-message units.
163
+ 3. The library-wide census figures (RSC-005 17,105 etc.) are therefore
164
+ distinct-defect-text counts. The true text-scale mass is higher
165
+ (vol3 alone contributes ~13k more locations than entries).
166
+
167
+ ### PKG-010: SOLVED. The mobi-rip split-chunk naming convention (wave 1, agent D)
168
+
169
+ 400-book sample (seed 42): **17 of 400 books (4.25%) carry space-bearing
170
+ zip entries**, projecting ~300 library-wide against the census's 287 (the
171
+ census value sits inside the binomial band). 872 space entries in the
172
+ sample, median 20 per book, one 591-entry outlier (The Red and the Black).
173
+ Extensions: .html 641 / .htm 223 / .xhtml 6, i.e. almost entirely split
174
+ content documents.
175
+
176
+ The decisive measurement: **the OPF hrefs match the zip names exactly**.
177
+ In all 10 books checked, every space-bearing href resolves to an existing
178
+ entry (Stars My Destination 20/20/20, Red and the Black 591/591/591).
179
+ Nothing is broken internally; the producing tool wrote consistent names.
180
+ Verbatim names fingerprint the pipeline: `Alfred Bester - Tiger Tiger!
181
+ (v1.2)_split_018.htm`, `Salvatore, RA - Cleric Quintet 1 -
182
+ Canticle_split_012.htm`, `Steven Brust - Vlad Taltos 02 - Yendi v4.0
183
+ (BD)_split_000.htm` — a mobi/azw3-to-epub rip pipeline that chunks
184
+ oversized source HTML into `<Author> - <Title>_split_NNN.htm` named after
185
+ the book's own metadata, plus a smaller Sigil-style subfamily
186
+ (`OEBPS/Text/Half Title.xhtml`).
187
+
188
+ Disposition: `--encode-url-spaces` already exists for exactly this (entry
189
+ renames to underscore spellings, every reference rewritten, ambiguous
190
+ renames refused, accepted under no_worse). A structural-flag pass over the
191
+ ~287-book population is the honest fix; the spaces are convention, not
192
+ loss.
193
+
194
+ ### RSC-007: TWO populations, both legacy-EPUB2 scars (wave 1, agent D)
195
+
196
+ 300-book sample: the xpgt hypothesis (H1) is **real but not the whole
197
+ story**.
198
+
199
+ 1. **The xpgt cluster**: 45 of 300 books reference `xpgt` (~46 refs per
200
+ referencing book), but 39 of them CARRY the template; only 6 of 300
201
+ reference it without carrying it, projecting ~141 of the 692 RSC-007
202
+ books. High mass though: 266 of 450 unresolved-reference occurrences in
203
+ the sample (59%) are xpgt (a handful of books at ~40-53 dangling refs
204
+ each, e.g. Global Crisis: 53, all `text/partNNNN.html ->
205
+ page-template.xpgt`). The reference is one shape everywhere:
206
+ `<link href="../Misc/page-template.xpgt" rel="stylesheet"
207
+ type="application/vnd.adobe-page-template+xml"/>` repeated in every
208
+ content doc, with converter-layout variants of the target path
209
+ (`Misc/`, `../styles/`, `ops/styles/`, `ArrowofGod/styles/`,
210
+ `page_template.xpgt`). This is the Adobe-era page template that a
211
+ converter dropped from the archive while leaving the per-document
212
+ links: **content-neutral loss** (the file never existed in the
213
+ archive, so no styling was ever applied), and `--prune-missing-resources`
214
+ is the honest repair for it: dropping dead `<link>` tags strictly
215
+ decreases the error count, normal gate.
216
+ 2. **The diffuse loss tail**: 50 of 300 books have at least one unresolved
217
+ internal reference (ceiling projection ~1,175 books, ~10.5k
218
+ occurrences; the counter did not skip `kindle:` URIs or fragment junk,
219
+ so treat as an upper bound). Median 2 per book: stripped image
220
+ sequences (`00012.jpg` x36 in one book), Calibre's own generated
221
+ stylesheets gone missing (`page_styles.css` x12, `stylesheet.css` x12),
222
+ missing covers (`cover.xhtml` x10), stray images/fonts, broken NCX
223
+ targets, malformed hrefs (concatenation artifacts like
224
+ `..._cop_r1.htmlop`). Case-by-case, not one flag.
225
+
226
+ Both populations are ~90-98% `package version="2.0"`: the damage was baked
227
+ in by the legacy conversion chain, and the modern EPUB3 portion of the
228
+ library is structurally clean. Control: How Music Works (9691) belongs to
229
+ neither class (zero space names, zero xpgt, zero missing hrefs), so the
230
+ scars are pipeline-specific, not library-wide.
231
+
232
+ Projected reconciliation of the census's 692 RSC-007 books: ~141 xpgt
233
+ books (high per-book mass) + the diffuse tail reproduces the total within
234
+ sampling error.
235
+
236
+ ### RSC-005: a narrow zoo behind a huge count (wave 1, agent A)
237
+
238
+ 20-book sample, epubcheck text output, reason texts normalized into
239
+ families. Total 129,312 RSC-005 **text occurrences** in 16 of 20 books, but
240
+ the sample is extremely skewed: four books carry 99.2% (Modern Japan
241
+ 76,859, Against the Gods 31,068, Rest Is Noise 12,658, Theory of Poker
242
+ 7,931). Sixteen of the 20 books are clean or near-clean on this code
243
+ (median ~28 per book). In aggregated-message terms (the census scale) this
244
+ sample's dominant families are exactly the ones that produce one entry per
245
+ distinct defect text.
246
+
247
+ | family | text occ | books | flag judgment |
248
+ |---|---|---|---|
249
+ | Duplicate id, 99.97% inside .svg files (`Duplicate "glNNNN"` gradient/filter ids, thousands per file) | 115,861 | 4 | `--fix-svg-dup-ids` (high confidence; built for this) |
250
+ | element not allowed here (content model; 97% is `span` directly in body under XHTML 1.1; also br/a/div/p/tbody; 2 OPF oddities: ns0:timestamp, ns1:title_sort) | 9,210 | 7 | `--reserialize` (medium; html5lib reparse makes body-level inline legal in HTML5). The OPF oddities: no flag |
251
+ | element incomplete (blockquote lacking a block child under XHTML 1.1) | 3,639 | 5 | `--reserialize` (same reasoning) |
252
+ | text not allowed here (bare text as body child) | 256 | 4 | `--reserialize` (medium-high) |
253
+ | id values with colons (`X:Y`) | 227 | 3 | `--fix-id-colons` (high; message names it verbatim) |
254
+ | attribute not allowed (deprecated: clear 51, charset 25, link 10, vlink 10, valign 5, border 1; OPF page-map 2; EPUB3 `properties` on an EPUB2 package 1) | 106 | 7 | `--strip-bad-attrs` (content docs), `--strip-epub3-attrs` (the properties hit) |
255
+ | invalid lang value | 5 | 1 | no honest flag (rewrite, not strip) |
256
+ | playOrder conflict (same target, two values) | 5 | 2 | no dedicated flag |
257
+ | pageList missing class; OPF missing nav property | 3 | 2 | no dedicated flag |
258
+
259
+ Takeaways: the RSC-005 mass is **duplicate SVG gradient ids in a handful
260
+ of conversion-heavy books**, and `--fix-svg-dup-ids` is the existing,
261
+ purpose-built repair. The XHTML 1.1 content-model families (span-in-body,
262
+ incomplete blockquote, bare body text) are the legacy-doctype shape that
263
+ `--reserialize` neutralizes by re-parsing as HTML5. Agent A's caveat: a
264
+ single heavily-SVG history book produced 60% of the sample volume, so the
265
+ library-wide family distribution is likely flatter; a stratified re-sample
266
+ off the v0.53.1 artifact's books array is queued as wave 2.
267
+
268
+ #### RSC-005 resample (wave 2): the typical book is epub3-semantics-on-legacy-doctype, not SVG
269
+
270
+ The 30-book stratified resample (Random(7), census pool minus the known
271
+ heavyweights) overturns the first sample's model:
272
+
273
+ - **The dominant family is EPUB3 semantics rejected by legacy DTDs**:
274
+ "attribute not allowed" = 10,403 of 12,364 resample occurrences across 18
275
+ books, led by `role` (6,645), `aria-label` (1,737), `epub:type` (1,453),
276
+ `aria-hidden` (239). One converter habit: modern semantic attributes on
277
+ HTML4/XHTML1 doctypes.
278
+ - The first sample's SVG duplicate-id family is **entirely absent** here
279
+ (0 of 12,364): duplicate ids are a per-book quirk of a handful of
280
+ SVG-annotated titles, contributing essentially nothing to the other
281
+ ~3,316 RSC-005 books.
282
+ - Mass concentration again, different books: five long-prose books (Some
283
+ Desperate Glory, House of Sky and Breath, Storming Heaven, The City and
284
+ Its Uncertain Walls, The Tiger) carry 91.5% of resample occurrences;
285
+ median 44; 13 of 30 books under 15. Zero of 30 books read clean, so the
286
+ census's RSC-005 book set is real.
287
+ - New metadata-level families the first sample missed: `@refines` missing
288
+ target id, invalid epub3 role value, href-not-a-URI,
289
+ Property-not-refining-creator, navList/pageList missing required
290
+ attribute. No flags.
291
+
292
+ **Flag gap (verified in source)**: `--strip-epub3-attrs`' fixed set is
293
+ exactly three attributes (`page-progression-direction`, `epub:type`,
294
+ `aria-label`). It covers aria-label and epub:type of the dominant family
295
+ but **not `role` or `aria-hidden`**, and its gate keys on the OPF package
296
+ version while epubcheck's rejection here is DOCTYPE-driven (an EPUB3
297
+ package carrying XHTML 1.1 doctypes would show the same findings yet be
298
+ skipped by the version gate). Extending the set and re-examining the gate
299
+ condition is the fix candidate for the family.
300
+
301
+ **Census-undercount confirmed systematically**: the artifact's per-book
302
+ error counts are unique-message-text counts; verified exactly on all 30
303
+ books, with undercount up to ~600x (one book: 5,946 occurrences collapse
304
+ to 10 census entries). Any repair prioritization must re-measure
305
+ occurrences directly rather than sort by the census field; the schema
306
+ revision (rename + a true occurrences count) is now load-bearing, not
307
+ cosmetic.
308
+
309
+ ### THE 313 REJECTS: SOLVED. Two transform bugs, un-masking killed (wave 1, agent C)
310
+
311
+ The 12-book message-diff surgery (before/after epubcheck on repairs
312
+ regenerated through the same code path the gate measures) kills the
313
+ un-masking hypothesis outright: **zero books had a masking construct** (no
314
+ prolog errors, no fatals, no prior playOrder findings), and every +delta is
315
+ a violation the fix itself wrote into previously clean bytes, verified at
316
+ byte level at identical coordinates. The gate's refusals were all correct:
317
+ 313 saves, not 313 misses.
318
+
319
+ **Bug 1: `fix_ncx_playorder` breaks contract-valid NCXs (199 of the 313
320
+ rejects).** The transform re-sequences `playOrder` across `<navPoint>` start
321
+ tags to document-order 1..N unconditionally. epubcheck's actual playOrder
322
+ contract: navPoints whose `<content src>` point at the same target must
323
+ SHARE one playOrder value; distinct targets need distinct values; no gaps,
324
+ and `pageTarget`/`navTarget` elements participate in the same sequence.
325
+ Real TOCs satisfy this with duplicate-target sharing (steel-breeze: two
326
+ navPoints at the same split_000.html, both `playOrder="1"`, zero findings
327
+ before, renumbered to 1/2 with a new finding at exactly those lines) and
328
+ with pageTargets interleaved (chronomancy: navPoints rewritten to 1,2,3
329
+ while the untouched `pageTarget ... playOrder="442"` at line 3034 now reads
330
+ as a sequence gap; mushroom-blues: 440 pageTargets, 0 -> 56 inconsistent
331
+ targets after the rewrite). Fix-mix evidence: 199 reject records carry
332
+ `fix_ncx_playorder` alone.
333
+
334
+ **Bug 2: `strip_prolog_junk` amputates the trailing `*/` of Calibre's CDATA
335
+ style wrappers (~110 of the 313).** `_PROTECTED_RE` matches
336
+ `<![CDATA[.*?]]>` non-greedily, so inside `/*<![CDATA[*/ ... /*]]>*/` the
337
+ protected span ends at the first `]]>` (inside the closing `/*]]>*/`),
338
+ leaving the trailing `*/` as the next outside segment. strip_prolog_junk
339
+ amputates everything before that segment's first `<`, eating `*/` plus
340
+ whitespace before `</style>` (the whitespace-only guard cannot catch `*`),
341
+ truncating the CSS comment, and epubcheck reports one CSS-008 "Premature
342
+ end of file" per broken file. Every fired instance in the sample is a
343
+ PHANTOM fix on a file with no prolog junk at all: fired counts equal
344
+ wrapper-file counts exactly (dark-star 18/18, sisyphus 6/6, calcutta 22/22,
345
+ moons-jupiter 21/21). At gate scale (aggregated entries) each book reads
346
+ +1.
347
+
348
+ **fix_named_entities: no breakage observed** (fired 3 times, all in one
349
+ book alongside the two buggy fixes; its mechanism cannot produce either
350
+ failure class; the same book's NCX-001 uid mismatch was genuinely repaired
351
+ by the uid sync, the one message-set loss in 12 books).
352
+
353
+ Consequences and scale note: both bugs sit in the DEFAULT pass (the NCX
354
+ pipeline is always-on; strip_prolog_junk is one of the five core
355
+ well-formedness fixes), so every library run attempts them; the gate is the
356
+ only thing that stood between the 199 books and a rewritten-but-worse NCX.
357
+ The aggregated-count scale also explains the tiny deltas: one bug hitting
358
+ 22 files reads as exactly +1. Evidence preserved in
359
+ `testing_facility/error-surgery-2026-10-24/rejects/` (before/after epubcheck
360
+ outputs, probe records, and five specimen books); an independent
361
+ claim-verification pass is in flight before any code change is proposed.
362
+
363
+ Candidate repairs (to be scoped after verification): fix_ncx_playorder
364
+ should only fire when the NCX actually violates the playOrder contract, and
365
+ its renumbering must share values across same-target navPoints and include
366
+ pageTarget/navTarget in the sequence; strip_prolog_junk needs the
367
+ protection span to cover the wrapper closer (`/*]]>*/` as a unit) or an
368
+ orphan-`*/` guard before the whitespace test.
369
+
370
+ #### Independent verification: 4/4 claims confirmed, zero refuted
371
+
372
+ The claim-verifier re-proved both bugs from scratch with minimal synthetic
373
+ reproducers and re-derived every count from the dry-run JSON:
374
+
375
+ - **Claim 1 CONFIRMED**: a minimal EPUB whose NCX shares `playOrder="1"`
376
+ across two navPoints on the same target validates clean; the transform
377
+ in isolation fires 2 (a phantom fix on a valid file), splits the shared
378
+ value to 1/2/3, and the repacked book fails with the exact claimed
379
+ finding, twice. Exactly 199 reject records carry
380
+ `{fix_ncx_playorder}` alone (deltas +1 x121, +2 x60, +3 x18); two of the
381
+ 313 rejects were 0f/0e fully clean books damaged by phantom fixes.
382
+ - **Claim 2 CONFIRMED**: isolation on a Calibre-wrapped inline style shows
383
+ the removed bytes are exactly `*/` (the style close becomes
384
+ `/*]]></style>`), the fired count is 1 on a file with no prolog junk,
385
+ and the repacked book fails with CSS-008 "Premature end of file".
386
+ 111 reject records contain strip_prolog_junk (73 solo + 29 with
387
+ fix_named_entities + 7 + 2 mixed).
388
+ - **Claim 3 CONFIRMED**: a 3-stylesheet demo shows 3 text occurrences
389
+ collapsing to 1 JSON entry; vol3 reproduces 20,307 text vs 7,030 JSON
390
+ exactly; the gate code reads `checker.nError` (validate.py).
391
+ - **Claim 4 CONFIRMED and strengthened**: fix_named_entities fired SOLO in
392
+ 93 records with zero rejections (14 accept at -1, 78 equal, 1 partial):
393
+ when alone it repairs or is neutral, never breaks. All 34 mixed records
394
+ are attributable to their companions.
395
+ - Verifier's aside (a third sequencing blind spot): a navPoint with no
396
+ playOrder attribute neither consumes nor receives a sequence number.
397
+ - Bookkeeping note: the facility's rejects/ dir holds 6 of the 12 sampled
398
+ books (the /tmp forensics set holds all 12); widen the facility set
399
+ before any fix session that wants the full spread.
400
+
401
+ #### The benefit ratio that decides the fix design
402
+
403
+ Across the entire dry run, fix_ncx_playorder fired on 579 books:
404
+
405
+ | outcome | books |
406
+ |---|---|
407
+ | equal (fired, zero count change: a no-op rewrite) | 347 (60%) |
408
+ | reject with positive delta (actively made worse) | 211 (36%) |
409
+ | accept with negative delta (the legitimate use) | **17 (3%)** |
410
+ | partial / error | 4 |
411
+
412
+ The transform helps 3% of the books it fires on and harms 36%. The
413
+ contract-gating repair (fire only when the NCX actually violates the
414
+ playOrder contract) turns the 347 no-ops and 211 harms into no-ops and
415
+ preserves the 17 real fixes.
416
+
417
+ ### Disposition probes: all three proposed flags CONFIRMED live (wave 2)
418
+
419
+ Each disposition was probed on a real library book (specimens in the
420
+ facility; gate records and before/after epubcheck transcripts preserved):
421
+
422
+ | class | book | flag | target delta | gate |
423
+ |---|---|---|---|---|
424
+ | RSC-005 svg dup ids | Making of Modern Japan | `--fix-svg-dup-ids` | 76,859 occurrences -> 0; **fully clean epubcheck** | accept, 104 svg entries touched, 71,711 ids fixed |
425
+ | RSC-007 xpgt dead links | Global Crisis | `--prune-missing-resources` | 53 RSC-007 -> 0; sole residual is an out-of-class RSC-020 mailto-space error present in the baseline | accept, dead_links_pruned=53 |
426
+ | PKG-010 spaces | Stars My Destination | `--encode-url-spaces` | 20 PKG-010 -> 0 AND 21 collateral RSC-020 -> 0; **fully clean epubcheck** | accept, 20 renames + 21 refs rewritten, zero rename-caused misses |
427
+
428
+ Two cross-cutting notes: the encode-url-spaces probe cleared the book's
429
+ entire RSC-020 population too (those were spaces-in-URL findings, the same
430
+ defect on the URL axis), so the two classes overlap on that shape; and
431
+ bindery's internal before-count for Modern Japan read 788 (aggregated
432
+ entries) against epubcheck's 76,859 (text locations) — the aggregation
433
+ scale again, consistent end to end.
434
+
435
+ ### RSC-020: SOLVED. The PKG-010 twin plus a junk-link tail (wave 2)
436
+
437
+ 15-book weighted sample off the artifact's books array (5
438
+ fingerprint-positive + 10 arbitrary; 178 raw lines, 127 unique URLs):
439
+
440
+ | family | raw lines | verdict |
441
+ |---|---|---|
442
+ | 1. space-in-URL, internal relative href, target exists | 133 (75%) | invalid-but-harmless naming; **the PKG-010 twin** |
443
+ | 2. escaped-markup/garbage href (`<i>` swallowed into an href, Windows path) | 20 | genuinely broken (missing internal targets) |
444
+ | 3. space-in-URL, external http(s) (line-wrap artifacts baked into hrefs) | 12 | genuinely broken addresses even after encoding |
445
+ | 4. %20 spliced mid-host | 7 | genuinely broken; encoding cannot fix (epubcheck percent-decodes then rejects the space) |
446
+ | 5. mailto-with-space | 2 | genuinely broken (unsendable) |
447
+ | 6. literal space in host | 1 | genuinely broken; still rejected if encoded |
448
+ | 7. illegal bracket in relative filename, target exists | 2 | harmless naming (needs %5B%5D or a rename, never a space flag) |
449
+ | 8. malformed host (empty domain label) | 1 | genuinely broken garbage |
450
+
451
+ Population overlap: **287 of the 353 RSC-020 books (81.3%) carry the
452
+ PKG-010 space-entry fingerprint**, and in every fingerprint-positive book
453
+ the counts are in strict three-way identity (PKG-010 count = zip
454
+ space-entry count = flagged internal space-URL count): one root cause, two
455
+ epubcheck layers. The `--encode-url-spaces` disposition therefore covers
456
+ family 1 wholesale (confirmed live on Stars My Destination, where all 21
457
+ RSC-020s cleared as collateral) and makes families 3 and 5 *parse* as
458
+ valid URLs while their targets stay dead (cosmetic repair at best). The
459
+ genuinely-broken families 2/4/6/8 live in the 66 fingerprint-negative
460
+ books, are near-zero user impact (junk links, mostly external), and have
461
+ no honest flag; leave them measured, not "repaired".
462
+
463
+ Counting-semantics proof from this lane: the artifact aggregates unique
464
+ flagged URLs per book (The Chill: 34 artifact vs 66 raw lines, each URL
465
+ flagged in both content.opf and toc.ncx), further confirming the
466
+ unique-text scale of every census number.
467
+
468
+ ### --reserialize: FALSIFIED for the content-model family, and the bug bites again (wave 2)
469
+
470
+ The Dark Star probe (the span-in-body exemplar) came back with two
471
+ structural findings:
472
+
473
+ 1. **The gate rejected the probe, and the regression is the prolog bug
474
+ again**: the fixes dict carries only `strip_prolog_junk: 18`, the
475
+ refusal is `0f/7e -> 0f/8e` (one aggregated CSS-008 entry), and a
476
+ control run without `--reserialize` produces a byte-identical reject
477
+ record. The core pass never let the opt-in stage run. The 18 CSS-008s
478
+ are exactly the class bindery's own `--fix-cdata-terminator` exists to
479
+ repair: the core pass manufactures work for an opt-in.
480
+ 2. **Even gate-free, reserialize is inert here**: its html5lib path
481
+ triggers only on XML ParseError ("runs only on documents that are not
482
+ already well-formed"), and the content-model family is
483
+ well-formed-but-schema-invalid. The no-validate hypothetical ran the
484
+ rebuild trigger zero times; all 273 RSC-005 content-model occurrences
485
+ survived; text preservation confirmed only that the strip damage lost
486
+ no prose.
487
+
488
+ The medium-confidence research claim is falsified for this exemplar: the
489
+ span-in-body / incomplete-blockquote / bare-body-text families have **no
490
+ working repair today**. Addressing them would mean a new opt-in that wraps
491
+ loose block content (or an explicit second reserialize trigger for
492
+ well-formed docs with content-model errors, contradicting its current
493
+ "good files are untouched" contract). Until then they stay measured, not
494
+ repaired.
495
+
496
+ ## Closing recommendations (2026-10-24 surgery)
497
+
498
+ 1. **Fix the two core-pass bugs first** (both in the always-on pass, both
499
+ gate-proven, independently verified with synthetic reproducers):
500
+ - `fix_ncx_playorder`: fire only when the NCX actually violates the
501
+ playOrder contract; share values across same-target navPoints;
502
+ include pageTarget/navTarget in the sequence; handle navPoints with
503
+ no playOrder attribute (the verifier's third blind spot). Benefit
504
+ ratio today: 17 helped / 211 harmed / 347 no-ops out of 579 firings.
505
+ - `strip_prolog_junk`: treat the Calibre wrapper `/*<![CDATA[*/ ...
506
+ /*]]>*/` as one protected unit (or guard the orphan `*/` before the
507
+ whitespace test). Today it phantom-fires on files with no prolog junk
508
+ and manufactures CSS-008 x N.
509
+ Re-run the whole-library dry run after the fix: the 313 rejects should
510
+ collapse to near-zero, and the 347/equal no-op rewrites should stop
511
+ firing.
512
+ 2. **Then the validated structural pass** (one reviewed apply run, same
513
+ backup pipeline as stage 2/2b): `--fix-svg-dup-ids` +
514
+ `--prune-missing-resources` + `--encode-url-spaces`. All three are
515
+ gate-proven to fully clear their classes on real specimens; the run
516
+ also clears RSC-020 family 1 as collateral. Ordering matters: the
517
+ structural pass must wait for the bug fixes because a book carrying
518
+ both the fake-CDATA idiom and a target defect would be refused
519
+ wholesale by the gate (the buggy core fix poisons the bundle).
520
+ 3. **Extend `--strip-epub3-attrs`** (`role`, `aria-hidden`) and re-examine
521
+ its package-version gate against the doctype-driven rejection, before
522
+ any pass aimed at the dominant RSC-005 family. Check the target
523
+ population's package versions first.
524
+ 4. **Schema revision for observe (v2)**: rename `codes.occurrences` ->
525
+ `codes.messages` (aggregated entries), add a true occurrences count
526
+ (locations), state the scale in the artifact and docs. Now load-bearing:
527
+ prioritization off per-book counts can mis-rank by ~600x.
528
+ 5. **No-repair list (leave measured)**: the Feynman equation scar
529
+ (re-source), the XHTML 1.1 content-model families (no working repair;
530
+ reserialize falsified), RSC-020 junk-link families 2/4/6/8, the
531
+ diffuse RSC-007 tail (case-by-case), the metadata-level RSC-005
532
+ families. These are the honest residue; the observatory tracks them.
533
+ 6. **Candidate new flag (low priority)**: flat-repack re-point (the
534
+ Gotrek & Felix cover-image class: the target exists under its flat
535
+ name; a basename-elsewhere re-point would fix 1-9-occurrence escapes).
536
+
537
+ Evidence: specimens and probe records under
538
+ `testing_facility/error-surgery-2026-10-24/` (rsc026/ rsc005/ rejects/
539
+ pkg010/), census artifacts under `~/docs/Calibre Library.backups/observe/`,
540
+ the dry-run JSON under the library's `.claude/`. All measurement was
541
+ read-only; the library was never written during this research.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: bindery-cli
3
- Version: 0.53.1
3
+ Version: 0.54.0
4
4
  Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
5
5
  Project-URL: Homepage, https://github.com/VirInvictus/bindery-cli
6
6
  Project-URL: Repository, https://github.com/VirInvictus/bindery-cli
@@ -1,5 +1,57 @@
1
1
  # bindery-cli Patch Notes
2
2
 
3
+ ## v0.54.0 (2026-10-24)
4
+
5
+ ### The error-code surgery fixes: the core pass stopped breaking valid books
6
+
7
+ The 2026-10-24 whole-library surgery (the record lives in
8
+ `OBSERVE-RESEARCH.md`, specimens in the gitignored testing facility)
9
+ proved that the tool's two worst whole-library behaviors, 313 gate
10
+ rejections and a wall of no-change rewrites, were both self-inflicted.
11
+ Two always-on core transforms carried bugs; the epubcheck gate correctly
12
+ refused every damaged output, which is why no library book was ever hurt.
13
+
14
+ **fix_ncx_playorder rewrote valid TOCs into violations.** The old form
15
+ renumbered every `<navPoint>` playOrder to document-order 1..N
16
+ unconditionally. epubcheck's actual contract is subtler: navPoints whose
17
+ `<content src>` point at the same target must SHARE one playOrder value,
18
+ distinct targets need distinct values, and pageTarget/navTarget elements
19
+ participate in the same contiguous sequence. Real TOCs satisfy this with
20
+ duplicate-target sharing and interleaved pageTargets, so the renumbering
21
+ manufactured RSC-005 playOrder findings on books whose NCX had zero
22
+ findings before. Whole-library tally: 579 firings, 17 real fixes, 211
23
+ gate-rejected regressions, 347 no-op rewrites. The transform now parses
24
+ the contract (element kind, playOrder value, content target in document
25
+ order), returns unchanged when the NCX already satisfies it, and on a
26
+ violation re-sequences by first-appearance target order, rewriting only
27
+ the attributes whose value changes. Verified against a minimal synthetic
28
+ EPUB: a shared-target NCX that validates clean goes in and comes out
29
+ untouched; the genuinely broken shapes (all-1 NCX, split shared values,
30
+ sequence gaps) still repair.
31
+
32
+ **strip_prolog_junk amputated the trailing `*/` of Calibre's CDATA style
33
+ wrappers.** The protection map's CDATA alternative ends at the first
34
+ `]]>`, which inside Calibre's `/*<![CDATA[*/ ... /*]]>*/` idiom sits
35
+ before the wrapper's own closing `*/`. That orphan landed in the outside
36
+ segment, the prolog strip ate it as "stray bytes before the first `<"`
37
+ (the whitespace guard cannot catch `*`), and every wrapped stylesheet
38
+ came back with a truncated comment: one CSS-008 "Premature end of file"
39
+ per file, a phantom fix whose fired count exactly equaled the wrapper
40
+ count (18/18, 22/22, 6/6 in the samples). The protection map now carries
41
+ the Calibre wrapper as its own first alternative, so the idiom is one
42
+ protected unit; a genuinely truncated closer (no trailing `*/`) stays
43
+ outside the span, where `--fix-cdata-terminator` can still reach it.
44
+
45
+ **fix_named_entities is exonerated** by the same surgery: it fired solo
46
+ in 93 whole-library records with zero rejections (14 real fixes, 78
47
+ count-neutral). No change.
48
+
49
+ Verification: 9-agent research pass with an independent claim-verification
50
+ lane (4/4 claims confirmed, minimal synthetic reproducers for both bugs);
51
+ 730 tests green (the playorder suite now pins the contract semantics,
52
+ including the shared-target regression test); a whole-library dry run
53
+ after the fix confirms the rejection collapse.
54
+
3
55
  ## v0.53.1 (2026-10-24)
4
56
 
5
57
  ### The .observe artifact now carries its per-book census rows
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "bindery-cli"
7
- version = "0.53.1"
7
+ version = "0.54.0"
8
8
  description = "Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library."
9
9
  readme = "README.md"
10
10
  classifiers = [
@@ -47,6 +47,7 @@ The plugin vendors the core and never gates (the epubcheck latency ruling).
47
47
  | 0.51 | The unbound-prefix repair (issue #9): `--fix-unbound-prefix` binds the Calibre embed scar's undeclared attribute prefix inline (evidence-based URI, scope-tracked, refuse over guess), plus the masked-ncx-gain rule that stops a real dtb:uid sync reading as no measurable gain under a fatal package | v0.51.0 |
48
48
  | 0.52 | The spine guard (issue #10): the manifest prune reads the layout its hrefs are written in and derives its spine protection from the text it prunes (the raw-space wave's spine collapse), and `process_book` vetoes any candidate whose spine (itemref -> item -> archive) dangles more than the source's, above the counts and every promotion | v0.52.0 |
49
49
  | 0.53 | `bindery observe` (the observatory): the whole-library census in one warm-JVM pass (counts and error codes together, sentinel-backed so no book is dropped), tabulation (severity histogram, code profile, worst offenders), the gated repair preview composed in-process, the versioned `.observe` artifact with `--against` trend diffing and the keep-root-clean output guard; the library and audit walks skip `.caltrash` | v0.53.0-v0.53.1 |
50
+ | 0.54 | The surgery fixes: `fix_ncx_playorder` is contract-gated (epubcheck's playOrder contract: same-target sharing, distinct-target distinctness, contiguous 1..K across navTarget/pageTarget; the old unconditional 1..N renumbering rewrote valid TOCs into violations, 211 of 579 firings gate-rejected vs 17 real fixes) and `_PROTECTED_RE` protects the Calibre `/*<![CDATA[*/ ... /*]]>*/` wrapper as one unit (the bare CDATA span exposed the trailing `*/` to strip_prolog_junk's amputation, 111 phantom-fix rejections); verified by the 2026-10-24 error-code surgery (OBSERVE-RESEARCH.md) | v0.54.0 |
50
51
 
51
52
  ## Open work
52
53
 
@@ -46,7 +46,12 @@ Applied to content documents (`.xhtml`, `.html`, `.htm`, `.xml`), in order:
46
46
  a declaration must be the very first thing in the document; before a DOCTYPE or the
47
47
  root element it is legal prolog whitespace and is left in place. (Stripping it
48
48
  anyway counted a fix on an undamaged document, which forced the archive rewrite's
49
- lossy `decode("utf-8", "replace")` round-trip for no gain.)
49
+ lossy `decode("utf-8", "replace")` round-trip for no gain.) Calibre's inline-CSS
50
+ wrapper `/*<![CDATA[*/ ... /*]]>*/` is protected as one unit (v0.54.0): the bare
51
+ CDATA span ends inside the closing `/*]]>`, which left the trailing `*/` outside
52
+ the protection map, where the prolog strip amputated it -- a phantom fix
53
+ truncating the CSS comment (one CSS-008 "Premature end of file" per wrapped
54
+ file; 111 whole-library gate rejections).
50
55
  2. **drop_duplicate_xmlns**: keep only the first `xmlns="..."` on the root `<html>`.
51
56
  3. **escape_bare_amp**: replace a `&` that does not start a valid entity or character
52
57
  reference with `&amp;`.
@@ -63,9 +68,17 @@ Applied to content documents (`.xhtml`, `.html`, `.htm`, `.xml`), in order:
63
68
  Applied to the NCX sidecar (`.ncx`): strip_prolog_junk, escape_bare_amp,
64
69
  fix_named_entities, plus **dtb:uid sync** (rewriting the NCX's `dtb:uid`
65
70
  meta to match the OPF's unique identifier when they drift, NCX-001) and
66
- **fix_ncx_playorder** (resequencing `playOrder` attributes on
67
- `<navPoint>` start tags so they are strictly sequential; nav-label text
68
- is never touched).
71
+ **fix_ncx_playorder** (resequencing `playOrder` values so the NCX
72
+ satisfies epubcheck's playOrder contract: elements whose `<content src>`
73
+ point at the same target share one value, distinct targets carry distinct
74
+ values, and the value set is contiguous 1..K across navPoint,
75
+ navTarget, and pageTarget elements together. Contract-gated since v0.54.0:
76
+ an NCX that already satisfies the contract is returned untouched -- the
77
+ previous unconditional 1..N renumbering rewrote valid TOCs into violations
78
+ (splitting duplicate-target sharing, colliding with pageTarget sequences;
79
+ 211 of the 579 books it fired on gate-rejected, against 17 real fixes).
80
+ nav-label text is never touched; a navPoint with no playOrder attribute is
81
+ not part of the sequence).
69
82
  With `--fix-ids`, ids in the NCX that are not valid XML names (digit-led, as when a
70
83
  converter stamps navPoint ids from UUIDs; colon-bearing) are renamed with the same
71
84
  `id_` scheme as OPF manifest ids. NCX ids are internal to the NCX (nothing in the
@@ -11,7 +11,7 @@ Calibre plugin built from this core.
11
11
  Single source of truth for the version. pyproject.toml must match VERSION below.
12
12
  """
13
13
 
14
- VERSION = "0.53.1"
14
+ VERSION = "0.54.0"
15
15
  __version__ = VERSION
16
16
 
17
17
  from .epub import RepairReport, repair_epub # noqa: E402
@@ -61,7 +61,18 @@ _EPUB_PREFIX_VAL_RE = re.compile(r"(?:^|\s)([A-Za-z_][\w.-]*)\s*:\s*\S+")
61
61
  # (e.g. `&` in CDATA-wrapped CSS/JS renders as `&`; escaped, it renders as `&amp;`).
62
62
  # Splitting on this regex yields alternating outside/protected segments, so the
63
63
  # markup transforms run only on the even (outside) indices.
64
- _PROTECTED_RE = re.compile(r"(<!\[CDATA\[.*?\]\]>|<!--.*?-->)", re.DOTALL)
64
+ #
65
+ # The first alternative is Calibre's inline-CSS wrapper idiom
66
+ # `/*<![CDATA[*/ ... /*]]>*/`: the bare CDATA alternative ends at the first `]]>`
67
+ # (inside the closing `/*]]>`), leaving the trailing `*/` as an outside segment,
68
+ # where strip_prolog_junk amputated it as "stray bytes before the first `<"` (a
69
+ # phantom fix that truncated the CSS comment and manufactured one CSS-008
70
+ # "Premature end of file" per wrapped file; the 2026-10-24 whole-library dry run
71
+ # gate-rejected 111 books on exactly this). The wrapper is protected as one unit.
72
+ _PROTECTED_RE = re.compile(
73
+ r"(/\*<!\[CDATA\[\*/.*?/\*\]\]>\*/|<!\[CDATA\[.*?\]\]>|<!--.*?-->)",
74
+ re.DOTALL,
75
+ )
65
76
 
66
77
 
67
78
  def outside_protected_map(s: str, fn: Transform) -> tuple[str, int]:
@@ -323,38 +334,127 @@ def drop_duplicate_xmlns(s: str) -> tuple[str, int]:
323
334
  return s, count
324
335
 
325
336
 
337
+ # The playOrder contract epubcheck enforces (the RSC-005 playOrder findings):
338
+ # elements whose <content src> point at the SAME target share one value,
339
+ # distinct targets carry distinct values, and the value set is contiguous
340
+ # 1..K across navPoint + navTarget + pageTarget elements together.
341
+ _PLAYORDER_EL_RE = re.compile(
342
+ r"<(?:navPoint|navTarget|pageTarget)\b[^>]*>", re.IGNORECASE
343
+ )
344
+ _PLAYORDER_ATTR_RE = re.compile(
345
+ r"\s+playOrder\s*=\s*(\"[^\"]*\"|'[^']*')", re.IGNORECASE
346
+ )
347
+ _CONTENT_SRC_RE = re.compile(
348
+ r"<content\b[^>]*?\ssrc\s*=\s*(\"[^\"]*\"|'[^']*')", re.IGNORECASE
349
+ )
350
+
351
+
326
352
  @_outside_protected
327
353
  def fix_ncx_playorder(s: str) -> tuple[str, int]:
328
- """Re-sequence `playOrder` integers across `<navPoint>` start tags so they
329
- are strictly sequential (NCX-004/NCX-005 class findings). The edit is
330
- anchored to navPoint start tags: the word `playOrder` inside a nav
331
- label's text is content, not markup, and is never touched.
354
+ """Re-sequence `playOrder` values so the NCX satisfies epubcheck's playOrder
355
+ contract, and touch nothing when it already does.
356
+
357
+ Contract-gated (the 2026-10-24 whole-library surgery): the previous form
358
+ renumbered every `<navPoint>` to document-order 1..N unconditionally, which
359
+ rewrote VALID TOCs into violations -- navPoints whose `<content src>` share
360
+ a target must share one playOrder (splitting them is a finding), pageTarget/
361
+ navTarget elements participate in the same sequence (ignoring them leaves
362
+ gaps), and the gate rejected 211 of the 579 books it fired on, against 17
363
+ real fixes. The resequencing here assigns each distinct target one value in
364
+ first-appearance order and rewrites only the attributes whose value changes.
365
+
366
+ Anchored to start tags of the three playOrder-bearing elements: the word
367
+ `playOrder` inside a nav label's text is content, not markup, and is never
368
+ touched. A navPoint with no playOrder attribute is not part of the sequence
369
+ (adding attributes is not this repair), and an element whose `<content src>`
370
+ cannot be located is treated as its own target. Known limit: the tag match
371
+ ends at the first `>`, so a `>` inside a quoted attribute value before the
372
+ playOrder attribute hides the element (the same single-class shape every
373
+ tag matcher here documents).
332
374
  """
333
- count = 0
334
- playorder = 1
375
+ elements: list[dict] = [] # (start, end, raw_value, target, order)
376
+ current: dict | None = None
377
+ for m in _PLAYORDER_EL_RE.finditer(s):
378
+ tag = m.group(0)
379
+ attr = _PLAYORDER_ATTR_RE.search(tag)
380
+ if attr is None:
381
+ continue
382
+ if current is not None:
383
+ elements.append(current)
384
+ current = {
385
+ "start": m.start(),
386
+ "end": m.end(),
387
+ "tag": tag,
388
+ "raw": attr.group(1),
389
+ "value": attr.group(1).strip("\"'"),
390
+ "target": None,
391
+ }
392
+ if current is not None:
393
+ elements.append(current)
394
+ if not elements:
395
+ return s, 0
335
396
 
336
- def repl(m: re.Match) -> str:
337
- nonlocal count, playorder
338
- # the captured value carries its quotes; strip them before comparing,
339
- # or every already-correct navPoint counts as a fix on every pass
340
- # (identical output bytes, phantom counts; found by the plugin work,
341
- # 2026-09-12). A correct value is returned untouched, so a
342
- # single-quoted correct attribute keeps its quote style.
343
- if m.group(2).strip("\"'") == str(playorder):
344
- res = m.group(0)
345
- else:
346
- count += 1
347
- res = f'{m.group(1)} playOrder="{playorder}"'
348
- playorder += 1
349
- return res
397
+ # Assign each element the src of the first <content> following its own start
398
+ # tag and before the next playOrder-bearing element starts (flat and
399
+ # parent-content-first nesting both assign correctly; children-content-first
400
+ # nesting is the documented limit).
401
+ for i, el in enumerate(elements):
402
+ next_start = elements[i + 1]["start"] if i + 1 < len(elements) else len(s)
403
+ m = _CONTENT_SRC_RE.search(s, el["end"], next_start)
404
+ if m is not None:
405
+ el["target"] = m.group(1).strip("\"'")
406
+
407
+ def key_of(el: dict) -> str:
408
+ # An element whose content src cannot be located is its own target
409
+ # (keyed by position so two such elements never merge).
410
+ return el["target"] if el["target"] is not None else f"\0{el['start']}"
411
+
412
+ groups: dict[str, list[dict]] = {}
413
+ for el in elements:
414
+ groups.setdefault(key_of(el), []).append(el)
415
+
416
+ # Contract check: every target group carries exactly one integer value,
417
+ # groups never share a value, and the value set is exactly 1..len(groups).
418
+ used: set[int] = set()
419
+ contract_ok = True
420
+ for members in groups.values():
421
+ vals: set[int] = set()
422
+ for el in members:
423
+ try:
424
+ vals.add(int(el["value"]))
425
+ except ValueError:
426
+ contract_ok = False
427
+ break
428
+ if not contract_ok or len(vals) != 1 or vals & used:
429
+ contract_ok = False
430
+ break
431
+ used |= vals
432
+ if contract_ok and used == set(range(1, len(groups) + 1)):
433
+ return s, 0
350
434
 
351
- s, n = re.subn(
352
- r"(<navPoint\b[^>]*?)\s+playOrder\s*=\s*(\"[^\"]*\"|'[^']*')",
353
- repl,
354
- s,
355
- flags=re.IGNORECASE,
356
- )
357
- return s, count
435
+ # Violating: re-sequence by first appearance of each target.
436
+ assigned: dict[str, int] = {}
437
+ out = []
438
+ last = 0
439
+ count = 0
440
+ for el in elements:
441
+ key = key_of(el)
442
+ if key not in assigned:
443
+ assigned[key] = len(assigned) + 1
444
+ new = str(assigned[key])
445
+ if el["value"] != new:
446
+ count += 1
447
+ attr = _PLAYORDER_ATTR_RE.search(el["tag"])
448
+ new_tag = (
449
+ el["tag"][: attr.start()]
450
+ + f' playOrder="{new}"'
451
+ + el["tag"][attr.end() :]
452
+ )
453
+ out.append(s[last : el["start"]])
454
+ out.append(new_tag)
455
+ last = el["end"]
456
+ out.append(s[last:])
457
+ return "".join(out), count
358
458
 
359
459
 
360
460
  @_outside_protected
@@ -288,6 +288,37 @@ class TestPrologAndXmlns(unittest.TestCase):
288
288
  out, n = strip_prolog_junk("<!DOCTYPE html><html/>")
289
289
  self.assertEqual((out, n), ("<!DOCTYPE html><html/>", 1))
290
290
 
291
+ def test_calibre_cdata_style_wrapper_is_never_amputated(self):
292
+ # the 2026-10-24 surgery: Calibre wraps inline CSS in
293
+ # /*<![CDATA[*/ ... /*]]>*/; the bare CDATA protection ended at the
294
+ # first ]]>, leaving the trailing */ as an outside segment that
295
+ # strip_prolog_junk amputated (a phantom fix truncating the CSS
296
+ # comment: one CSS-008 "Premature end of file" per wrapped file,
297
+ # 111 gate-rejected books library-wide)
298
+ s = (
299
+ '<?xml version="1.0"?><html><head><style type="text/css">'
300
+ "/*<![CDATA[*/ p { color: red } /*]]>*/</style></head>"
301
+ "<body/></html>"
302
+ )
303
+ self.assertEqual(strip_prolog_junk(s), (s, 0))
304
+ self.assertIn("/*]]>*/", s)
305
+
306
+ def test_truncated_calibre_wrapper_still_reachable_for_the_terminator_fix(self):
307
+ # a genuinely truncated closer (no trailing */) must stay outside the
308
+ # protected span so --fix-cdata-terminator can still see and repair it
309
+ s = (
310
+ '<?xml version="1.0"?><html><head><style type="text/css">'
311
+ "/*<![CDATA[*/ p { color: red } /*]]></style></head>"
312
+ "<body/></html>"
313
+ )
314
+ self.assertEqual(strip_prolog_junk(s), (s, 0))
315
+ self.assertNotIn("/*]]>*/", s) # the truncated shape, preserved as-is
316
+
317
+ def test_real_junk_still_stripped(self):
318
+ out, n = strip_prolog_junk("\xef\xbb\xbfstray bytes<html/>")
319
+ self.assertEqual(n, 1)
320
+ self.assertTrue(out.startswith("<html/>"))
321
+
291
322
  def test_junk_stripped_without_a_declaration(self):
292
323
  out, n = strip_prolog_junk("stray<html/>")
293
324
  self.assertEqual((out, n), ("<html/>", 1))
@@ -530,12 +561,87 @@ class TestFixNcxPlayorder(unittest.TestCase):
530
561
  self.assertEqual(out, ncx)
531
562
 
532
563
  def test_resequence_count_still_counts_real_changes(self):
564
+ # contract semantics (2026-10-24): two playOrder-bearing elements with
565
+ # no <content> are distinct targets, so sharing one value violates the
566
+ # distinctness rule; resequencing changes only the first (1) and the
567
+ # second lands on 2 unchanged, so one attribute change
533
568
  ncx = '<navPoint playOrder="2"/><navPoint playOrder="2"/>'
534
569
  out, n = fix_ncx_playorder(ncx)
535
- self.assertEqual(n, 1) # the first moves 2->1; the second is right
570
+ self.assertEqual(n, 1)
536
571
  self.assertIn('playOrder="1"', out)
537
572
  self.assertIn('playOrder="2"', out)
538
573
 
574
+ def test_shared_target_valid_ncx_is_never_rewritten(self):
575
+ # the steel-breeze shape from the 2026-10-24 surgery: two navPoints
576
+ # whose <content src> point at the same file legitimately share one
577
+ # playOrder; the old unconditional renumbering split them into a
578
+ # finding and the gate rejected the book
579
+ ncx = (
580
+ '<navPoint id="num_1" playOrder="1"><content src="Text/split_000.html"/></navPoint>'
581
+ '<navPoint id="num_2" playOrder="1"><content src="Text/split_000.html"/></navPoint>'
582
+ )
583
+ out, n = fix_ncx_playorder(ncx)
584
+ self.assertEqual((out, n), (ncx, 0))
585
+
586
+ def test_pagetarget_participates_in_the_sequence(self):
587
+ # chronomancy's shape: valid navPoints 1..3 with a pageTarget pointing
588
+ # at its own page far ahead is a sequence gap; the pageTarget joins
589
+ # the sequence as value 4
590
+ ncx = (
591
+ '<navPoint playOrder="1"><content src="a.xhtml"/></navPoint>'
592
+ '<navPoint playOrder="2"><content src="b.xhtml"/></navPoint>'
593
+ '<navPoint playOrder="3"><content src="c.xhtml"/></navPoint>'
594
+ '<pageTarget id="page_447" value="447" playOrder="442">'
595
+ '<content src="page447.xhtml"/></pageTarget>'
596
+ )
597
+ out, n = fix_ncx_playorder(ncx)
598
+ self.assertEqual(n, 1)
599
+ self.assertIn('playOrder="4"', out)
600
+ self.assertNotIn('playOrder="442"', out)
601
+
602
+ def test_pagetarget_sequence_valid_is_untouched(self):
603
+ ncx = (
604
+ '<navPoint playOrder="1"><content src="a.xhtml"/></navPoint>'
605
+ '<navPoint playOrder="2"><content src="b.xhtml"/></navPoint>'
606
+ '<pageTarget id="pg" value="3" playOrder="3">'
607
+ '<content src="c.xhtml"/></pageTarget>'
608
+ )
609
+ out, n = fix_ncx_playorder(ncx)
610
+ self.assertEqual((out, n), (ncx, 0))
611
+
612
+ def test_same_target_conflicting_values_share_one_value(self):
613
+ ncx = (
614
+ '<navPoint playOrder="1"><content src="a.xhtml"/></navPoint>'
615
+ '<navPoint playOrder="2"><content src="a.xhtml"/></navPoint>'
616
+ '<navPoint playOrder="3"><content src="b.xhtml"/></navPoint>'
617
+ )
618
+ out, n = fix_ncx_playorder(ncx)
619
+ # groups in first-appearance order: a.xhtml -> 1 (shared by the first
620
+ # two), b.xhtml -> 2; both the conflict (2) and the old 3 are rewritten
621
+ self.assertEqual(n, 2)
622
+ self.assertEqual(out.count('playOrder="1"'), 2)
623
+ self.assertIn('playOrder="2"', out)
624
+ self.assertNotIn('playOrder="3"', out)
625
+
626
+ def test_nested_navpoints_with_parent_content_first_are_untouched(self):
627
+ ncx = (
628
+ '<navPoint id="a" playOrder="1"><navLabel><text>A</text></navLabel>'
629
+ '<content src="a.xhtml"/>'
630
+ '<navPoint id="b" playOrder="2"><navLabel><text>B</text></navLabel>'
631
+ '<content src="b.xhtml"/></navPoint></navPoint>'
632
+ )
633
+ out, n = fix_ncx_playorder(ncx)
634
+ self.assertEqual((out, n), (ncx, 0))
635
+
636
+ def test_non_integer_playorder_is_resequenced(self):
637
+ ncx = (
638
+ '<navPoint playOrder="x"><content src="a.xhtml"/></navPoint>'
639
+ '<navPoint playOrder="2"><content src="b.xhtml"/></navPoint>'
640
+ )
641
+ out, n = fix_ncx_playorder(ncx)
642
+ self.assertEqual(n, 1)
643
+ self.assertIn('playOrder="1"', out)
644
+
539
645
 
540
646
  class TestStripInvalidValue(unittest.TestCase):
541
647
  def test_misplaced_value_is_stripped(self):
@@ -8,7 +8,7 @@ resolution-markers = [
8
8
 
9
9
  [[package]]
10
10
  name = "bindery-cli"
11
- version = "0.53.1"
11
+ version = "0.54.0"
12
12
  source = { editable = "." }
13
13
  dependencies = [
14
14
  { name = "cquarry", marker = "python_full_version >= '3.14'" },
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes