bindery-cli 0.53.0__tar.gz → 0.54.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/AGENTS.md +1 -1
- bindery_cli-0.54.0/OBSERVE-RESEARCH.md +541 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/PKG-INFO +1 -1
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/patchnotes.md +65 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/pyproject.toml +1 -1
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/roadmap.md +2 -1
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/spec.md +17 -4
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/__init__.py +1 -1
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/observe.py +13 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/transforms.py +128 -28
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_observe.py +7 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_transforms.py +107 -1
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/uv.lock +1 -1
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/.github/FUNDING.yml +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/.github/workflows/ci.yml +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/.github/workflows/publish.yml +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/.gitignore +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/LICENSE +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/README.md +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/docs/screenshots/library-sweep.png +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/logo.svg +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/plugin/__init__.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/project.done +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/run_tests.sh +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/scripts/FastSweep.java +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/scripts/build_plugin.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/scripts/fast_sweep.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/__main__.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/audit.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/cli.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/epub.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/library.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/pagination.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/reserialize.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/validate.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/src/bindery/watermark.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_audit.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_cli.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_comment_hyphens.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_contamination.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_cover_tocdrift.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_epub.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_flags_real_core.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_flags_wiring.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_help.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_library.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_matcher_hardening.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_pagination.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_plugin.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_reserialize.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_space_renames.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_spine_guard.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_validate.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_version.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_watermark.py +0 -0
- {bindery_cli-0.53.0 → bindery_cli-0.54.0}/tests/test_xml_hardening.py +0 -0
|
@@ -132,7 +132,7 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
|
|
|
132
132
|
|
|
133
133
|
## Layout
|
|
134
134
|
|
|
135
|
-
- `src/bindery/transforms.py`: pure `str -> (str, int)` text transforms (including `strip_broken_tags`). Since v0.32.0: `strip_attrs_in_start_tags` anchors attribute edits on the quote-aware start-tag matcher, `_outside_protected` forwards arguments (context-carrying transforms can be decorated), `fix_id_colons` matches only the bare `id` attribute and only internal fragments (`fix_ncx_src_fragments` carries renames into the NCX), `fix_ncx_playorder` is anchored to `<navPoint>` start tags, and the css selector boundary covers namespaced and functional selector forms. HAZARD (the 0.41.0 spin): the quote-aware tag-prefix alternation (double-quoted | single-quoted | [^>]) OVERLAPS ([^>] matches quotes too), so a plain `*` re-partitions exponentially on failing candidates; every such greedy loop must carry the possessive `*+` (re 3.11+, the plugin's minimum Calibre). Since v0.42.0 the census is complete and pinned by tests/test_matcher_hardening.py: every greedy loop of this family is possessive (the three 0.41.1 rewrites plus _START_TAG_RE, _COVER_META_RE, _GUIDE_REF_RE, prune_dangling_edges' spine|item|itemref matcher, _ITEM_TAG_RE, _SPINE_ITEM_RE; possessive is byte-identical on success because the loop is always followed by a literal `>` the [^>] branch cannot consume), and the five LAZY siblings (_VOID_RE, the transforms <img> matcher, and epub's link/a/img matchers) use the disjoint catch-all `[^>"']` instead: mutually exclusive branches cannot re-partition, and an unterminated quote inside a tag now fails to match rather than pairing across markup. NEW tag matchers: quote-aware possessive for greedy loops, the disjoint catch-all for lazy ones, and the tempered-quote idiom (["'])((?:(?!\1).)*)\1 for attribute values. Since v0.37.0: `fix_ncx_playorder` strips quotes before its already-correct comparison (regex groups carry quote characters; comparing quoted values against bare numbers counted phantom fixes on every sequential NCX and rewrote single-quoted correct attributes to double quotes).
|
|
135
|
+
- `src/bindery/transforms.py`: pure `str -> (str, int)` text transforms (including `strip_broken_tags`). Since v0.32.0: `strip_attrs_in_start_tags` anchors attribute edits on the quote-aware start-tag matcher, `_outside_protected` forwards arguments (context-carrying transforms can be decorated), `fix_id_colons` matches only the bare `id` attribute and only internal fragments (`fix_ncx_src_fragments` carries renames into the NCX), `fix_ncx_playorder` is anchored to `<navPoint>` start tags, and the css selector boundary covers namespaced and functional selector forms. HAZARD (the 0.41.0 spin): the quote-aware tag-prefix alternation (double-quoted | single-quoted | [^>]) OVERLAPS ([^>] matches quotes too), so a plain `*` re-partitions exponentially on failing candidates; every such greedy loop must carry the possessive `*+` (re 3.11+, the plugin's minimum Calibre). Since v0.42.0 the census is complete and pinned by tests/test_matcher_hardening.py: every greedy loop of this family is possessive (the three 0.41.1 rewrites plus _START_TAG_RE, _COVER_META_RE, _GUIDE_REF_RE, prune_dangling_edges' spine|item|itemref matcher, _ITEM_TAG_RE, _SPINE_ITEM_RE; possessive is byte-identical on success because the loop is always followed by a literal `>` the [^>] branch cannot consume), and the five LAZY siblings (_VOID_RE, the transforms <img> matcher, and epub's link/a/img matchers) use the disjoint catch-all `[^>"']` instead: mutually exclusive branches cannot re-partition, and an unterminated quote inside a tag now fails to match rather than pairing across markup. NEW tag matchers: quote-aware possessive for greedy loops, the disjoint catch-all for lazy ones, and the tempered-quote idiom (["'])((?:(?!\1).)*)\1 for attribute values. Since v0.37.0: `fix_ncx_playorder` strips quotes before its already-correct comparison (regex groups carry quote characters; comparing quoted values against bare numbers counted phantom fixes on every sequential NCX and rewrote single-quoted correct attributes to double quotes). Since v0.54.0 (the 2026-10-24 surgery): `fix_ncx_playorder` is CONTRACT-GATED -- it tokenizes navPoint/navTarget/pageTarget start tags plus their `<content src>` targets, returns unchanged when the NCX already satisfies epubcheck's playOrder contract (same target shares one value, distinct targets distinct values, values contiguous 1..K), and re-sequences by first-appearance target order only on a violation; the old unconditional 1..N renumbering rewrote valid TOCs into violations (211 of 579 firings gate-rejected, 17 real fixes, 347 no-ops). The same fix extends `_PROTECTED_RE` with the Calibre `/*<![CDATA[*/ ... /*]]>*/` wrapper as one protected unit (the bare CDATA alternative ends inside the closing `/*]]>`, which exposed the trailing `*/` to strip_prolog_junk's amputation: 111 phantom-fix rejections).
|
|
136
136
|
- `src/bindery/pagination.py`: the opt-in lossy page-number strip (runhead detection, page-layer decision, block-centric removal/merge, safety nets).
|
|
137
137
|
- `src/bindery/watermark.py`: the opt-in lossy watermark strip (anchored and anchorless signature removal).
|
|
138
138
|
- `src/bindery/reserialize.py`: structural repair via `html5lib`.
|
|
@@ -0,0 +1,541 @@
|
|
|
1
|
+
# Observe research: the error-code surgery (2026-10-24)
|
|
2
|
+
|
|
3
|
+
Working research file behind bindery observe's first library census
|
|
4
|
+
(`~/docs/Calibre Library.backups/observe/observe-2026-10-24-base.observe`).
|
|
5
|
+
Question: what exactly is triggering the library's four dominant epubcheck
|
|
6
|
+
code classes, and what exactly makes the gate reject the 313 repairs it
|
|
7
|
+
refused on the whole-library dry run? Specimen books live in
|
|
8
|
+
`testing_facility/error-surgery-2026-10-24/` (gitignored; never modified
|
|
9
|
+
copies of library books). This file is the living record; findings land here
|
|
10
|
+
as the investigation proceeds.
|
|
11
|
+
|
|
12
|
+
## The oracle's own definitions (epubcheck 5.3.0 MessageBundle.properties)
|
|
13
|
+
|
|
14
|
+
| Code | epubcheck's message | Library mass |
|
|
15
|
+
|---|---|---|
|
|
16
|
+
| RSC-005 | `Error while parsing file: %1$s` (the generic XML parse-error class; a zoo of distinct underlying defects) | 17,105 occurrences in 3,323 books |
|
|
17
|
+
| RSC-007 | `Referenced resource "%1$s" could not be found in the EPUB.` | 14,341 in 692 books (~20.7/book) |
|
|
18
|
+
| RSC-026 | `URL "%1$s" leaks outside the container (it is not a valid-relative-ocf-URL-with-fragment string)` | 12,607 in only 18 books (~700/book) |
|
|
19
|
+
| RSC-020 | `"%1$s" is not a valid URL (%2$s)` | 10,102 in 353 books |
|
|
20
|
+
| PKG-010 | `The file name "%1$s" contains spaces, which may create interoperability issues with older reading systems.` | 8,294 in 287 books (~29/book) |
|
|
21
|
+
|
|
22
|
+
Siblings in the top ten: RSC-001 (`File "%1$s" could not be found`, 1,546 in
|
|
23
|
+
39 books), HTM-004 (irregular DOCTYPE, in the Wandering Inn signature),
|
|
24
|
+
RSC-008 (referenced resource not declared in the OPF manifest), CSS-007,
|
|
25
|
+
RSC-004 (encrypted files), OPF-029/030.
|
|
26
|
+
|
|
27
|
+
## The structural fact that frames everything
|
|
28
|
+
|
|
29
|
+
The whole-library preview ran the DEFAULT flag set (the five always-on
|
|
30
|
+
well-formedness fixes plus the NCX pipeline and mimetype repair). Every fix
|
|
31
|
+
that addresses the four classes above is opt-in:
|
|
32
|
+
|
|
33
|
+
- RSC-007 -> `--prune-missing-resources` (drops references to absent files)
|
|
34
|
+
- PKG-010 -> `--encode-url-spaces` (renames the entries and rewrites every
|
|
35
|
+
reference: OPF, NCX, content, CSS url())
|
|
36
|
+
- RSC-005 families -> spread across `--fix-ids`, `--strip-bad-attrs`,
|
|
37
|
+
`--strip-epub3-attrs`, `--fix-misnested-inline`, `--fix-id-colons`,
|
|
38
|
+
`--fix-unterminated-attr`, `--reserialize`, and friends
|
|
39
|
+
- RSC-026 / RSC-020 -> no dedicated repair today (candidate analysis below)
|
|
40
|
+
|
|
41
|
+
So the 3,156 `nochange` books are mostly books whose defects belong to
|
|
42
|
+
opt-in classes the default pass never attempts. That is by design, not a
|
|
43
|
+
failure. The 313 `reject`s are the interesting anomaly: there the DEFAULT
|
|
44
|
+
fixes fired and the gate refused because the error count went UP.
|
|
45
|
+
|
|
46
|
+
## The 313 rejects: first shape
|
|
47
|
+
|
|
48
|
+
From the 2026-10-24 dry run (`bindery-dryrun-2026-10-24.json`):
|
|
49
|
+
|
|
50
|
+
- fix mix: `fix_ncx_playorder`-only 199, `strip_prolog_junk`-only 73,
|
|
51
|
+
`fix_named_entities`+`strip_prolog_junk` 29, mixes 12
|
|
52
|
+
- delta shapes: overwhelmingly +1 or +2 errors ((2,4) x50, (1,2) x42,
|
|
53
|
+
(2,3) x35, (5,6) x29, (4,5) x25, (3,4) x19)
|
|
54
|
+
|
|
55
|
+
Working hypothesis (to be proven or killed by the message-diff surgery):
|
|
56
|
+
the repair un-masks latent errors. epubcheck's parser stops counting at a
|
|
57
|
+
malformed construct; when the default pass removes junk before the XML
|
|
58
|
+
prolog, normalizes entities, or renumbers the NCX, validation reaches
|
|
59
|
+
deeper into the file and surfaces errors that were always there. The gate
|
|
60
|
+
sees a count increase on a no-fatal book and refuses, which is correct
|
|
61
|
+
behavior for a `strict error decrease` bar, but the books' TRUE defect set
|
|
62
|
+
is unchanged. The question the message diffs answer: does the delta
|
|
63
|
+
consist of NEW message families (un-masking) or more of the same message
|
|
64
|
+
(the fix genuinely breaking something)?
|
|
65
|
+
|
|
66
|
+
## Surgery log
|
|
67
|
+
|
|
68
|
+
### RSC-026: SOLVED. The `/root/work/img` equation scar (wave 1, agent B)
|
|
69
|
+
|
|
70
|
+
Specimen: the three Feynman Lectures volumes (facility copies,
|
|
71
|
+
`testing_facility/error-surgery-2026-10-24/rsc026/`). On vol3, **100% of
|
|
72
|
+
RSC-026 occurrences (10,044) are `<img src="/root/work/img/<32-hex>.png">`**
|
|
73
|
+
tags sitting mid-sentence where inline equations render. Zero absolute
|
|
74
|
+
http(s) used internally, zero `file://`, zero `../` escapes, zero
|
|
75
|
+
percent-encoding.
|
|
76
|
+
|
|
77
|
+
Mechanism: an upstream conversion pipeline ran as root with a scratch
|
|
78
|
+
workdir `/root/work`, rendered every equation and figure to a PNG named by
|
|
79
|
+
its md5, and emitted root-absolute `src` references. On the build machine
|
|
80
|
+
those paths resolved, so the conversion "succeeded" and the HTML was zipped
|
|
81
|
+
without any packager ever rewriting or embedding the images (Calibre embeds
|
|
82
|
+
only images it can find relative to a source tree it knows about;
|
|
83
|
+
root-absolute refs into its own temp dir are invisible once that dir is
|
|
84
|
+
gone). `class="calibre10"` fingerprints the Calibre pass-through. The
|
|
85
|
+
3,499 distinct URLs are uniformly-random md5s (3,414 distinct 4-hex
|
|
86
|
+
prefixes), i.e. content hashes, not sequence numbers; every one of the 21
|
|
87
|
+
content docs is affected (top: 902, 801, 791 occurrences per file).
|
|
88
|
+
|
|
89
|
+
Two compounding facts:
|
|
90
|
+
|
|
91
|
+
1. **RSC-026 and RSC-007 are 1:1 twins for the same defect** on this book:
|
|
92
|
+
the URL is invalid (leaks the container) AND the resource is missing.
|
|
93
|
+
On vol3: RSC-026 10,044 text occurrences / RSC-007 10,106, of which
|
|
94
|
+
10,044 are the same equation images. Together they are 99.2% of vol3's
|
|
95
|
+
error mass. The remaining 62 RSC-007s are a separate small habit:
|
|
96
|
+
hand-written cross-volume chapter links (`I_37.html`, `II_34.html`) that
|
|
97
|
+
assume the other volumes' files live in the same package.
|
|
98
|
+
2. **No repair in bindery's inventory should touch this.**
|
|
99
|
+
`--prune-missing-resources` would delete the references, which means
|
|
100
|
+
deleting every equation in a physics text: content destruction, not
|
|
101
|
+
repair. These books need re-sourcing (the census says so; the flag
|
|
102
|
+
inventory has nothing honest to offer). The 18-book RSC-026 population
|
|
103
|
+
(12,607 aggregated messages) is almost certainly all the same producer
|
|
104
|
+
fingerprint (root-absolute md5 workdir images); confirm from the books
|
|
105
|
+
array when the v0.53.1 baseline lands.
|
|
106
|
+
|
|
107
|
+
#### RSC-026 population revised (wave 2): the 18 carriers are NOT one producer
|
|
108
|
+
|
|
109
|
+
The fingerprint check across 6 of the 15 non-Feynman carriers (Dawn of
|
|
110
|
+
Everything, Embracing Defeat, Feet of Clay, all three Gotrek & Felix
|
|
111
|
+
omnibuses; specimens in the facility) kills the one-fingerprint
|
|
112
|
+
generalization: zero `/root/work` literals anywhere outside Feynman.
|
|
113
|
+
RSC-026 is epubcheck's generic "URL escapes the container" code, so it
|
|
114
|
+
aggregates unrelated defects:
|
|
115
|
+
|
|
116
|
+
- **Pipeline A (Feynman x3)**: the root-absolute md5 equation images. This
|
|
117
|
+
is the entire mass (12,607 aggregated messages library-wide is
|
|
118
|
+
essentially all Feynman).
|
|
119
|
+
- **Pipeline B (Calibre flat EPUB2 repacks)**: HTML flattened to zip root
|
|
120
|
+
while refs kept pre-flatten `../` prefixes. Three sub-shapes: cover img
|
|
121
|
+
`../Images/0000NN.jpg` (G&F family, byte-identical template, plus a case
|
|
122
|
+
mismatch: the dir is lowercase `images/`), content img `../images/...`
|
|
123
|
+
(Dawn of Everything), nav anchor `../Text/...#fileposNNN` (Feet of Clay).
|
|
124
|
+
Tiny counts (1-9 per book). Notably REPAIRABLE in principle: the G&F and
|
|
125
|
+
Dawn images exist in the archive under their flat names, so a re-point
|
|
126
|
+
(basename exists elsewhere in the archive) would fix them; that is a
|
|
127
|
+
candidate repair class, not an existing flag.
|
|
128
|
+
- **Pipeline C (Embracing Defeat)**: 9 garbage hrefs `../../../../l%20` —
|
|
129
|
+
broken source-conversion cross-references; strip-broken-anchors class.
|
|
130
|
+
|
|
131
|
+
Also surfaced: Dawn of Everything carries ~13,490 RSC-005 text
|
|
132
|
+
occurrences, another heavyweight outside agent A's sample; the stratified
|
|
133
|
+
resample matters.
|
|
134
|
+
|
|
135
|
+
### MEASUREMENT FINDING: the census counts aggregated messages, not text-summary occurrences
|
|
136
|
+
|
|
137
|
+
Direct run on vol3: the epubcheck **text summary** says `20,307 errors`;
|
|
138
|
+
the epubcheck **`--json`** (and identically `CheckingReport.generate()`,
|
|
139
|
+
which the census harness and the gate both read) says **`nError: 7030`**.
|
|
140
|
+
Both are real; they are different units:
|
|
141
|
+
|
|
142
|
+
- JSON: one **message entry** per distinct defect (ID + message text),
|
|
143
|
+
carrying a `locations` array with every occurrence site (plus
|
|
144
|
+
`additionalLocations`). vol3: RSC-007 = 3,528 entries, RSC-026 = 3,499
|
|
145
|
+
entries (exactly the 3,499 distinct equation URLs), RSC-005 = 3 entries
|
|
146
|
+
(the `valign` family), total 7,030.
|
|
147
|
+
- Text summary: sums the **locations** (20,307 = every site individually).
|
|
148
|
+
|
|
149
|
+
Consequences:
|
|
150
|
+
|
|
151
|
+
1. The gate and the whole observe stack are internally consistent: before,
|
|
152
|
+
after, the dry run, the preview, and the census all read
|
|
153
|
+
`checker.nError`, the same aggregated scale. No accept/reject verdict
|
|
154
|
+
changes.
|
|
155
|
+
2. The `.observe` artifact's `codes.occurrences` field is **mislabeled**:
|
|
156
|
+
it counts aggregated message entries (distinct defect texts), not
|
|
157
|
+
occurrences. Comparing it to a manual `epubcheck` text run will
|
|
158
|
+
understate by the aggregation factor (vol3: 7,030 vs 20,307, 2.9x).
|
|
159
|
+
The field should be renamed (`codes.messages`) and/or paired with a
|
|
160
|
+
true occurrences count (sum of `locations` lengths), at the next schema
|
|
161
|
+
revision. Until then: every number in this report and in the census
|
|
162
|
+
that came from the JSON path is in aggregated-message units.
|
|
163
|
+
3. The library-wide census figures (RSC-005 17,105 etc.) are therefore
|
|
164
|
+
distinct-defect-text counts. The true text-scale mass is higher
|
|
165
|
+
(vol3 alone contributes ~13k more locations than entries).
|
|
166
|
+
|
|
167
|
+
### PKG-010: SOLVED. The mobi-rip split-chunk naming convention (wave 1, agent D)
|
|
168
|
+
|
|
169
|
+
400-book sample (seed 42): **17 of 400 books (4.25%) carry space-bearing
|
|
170
|
+
zip entries**, projecting ~300 library-wide against the census's 287 (the
|
|
171
|
+
census value sits inside the binomial band). 872 space entries in the
|
|
172
|
+
sample, median 20 per book, one 591-entry outlier (The Red and the Black).
|
|
173
|
+
Extensions: .html 641 / .htm 223 / .xhtml 6, i.e. almost entirely split
|
|
174
|
+
content documents.
|
|
175
|
+
|
|
176
|
+
The decisive measurement: **the OPF hrefs match the zip names exactly**.
|
|
177
|
+
In all 10 books checked, every space-bearing href resolves to an existing
|
|
178
|
+
entry (Stars My Destination 20/20/20, Red and the Black 591/591/591).
|
|
179
|
+
Nothing is broken internally; the producing tool wrote consistent names.
|
|
180
|
+
Verbatim names fingerprint the pipeline: `Alfred Bester - Tiger Tiger!
|
|
181
|
+
(v1.2)_split_018.htm`, `Salvatore, RA - Cleric Quintet 1 -
|
|
182
|
+
Canticle_split_012.htm`, `Steven Brust - Vlad Taltos 02 - Yendi v4.0
|
|
183
|
+
(BD)_split_000.htm` — a mobi/azw3-to-epub rip pipeline that chunks
|
|
184
|
+
oversized source HTML into `<Author> - <Title>_split_NNN.htm` named after
|
|
185
|
+
the book's own metadata, plus a smaller Sigil-style subfamily
|
|
186
|
+
(`OEBPS/Text/Half Title.xhtml`).
|
|
187
|
+
|
|
188
|
+
Disposition: `--encode-url-spaces` already exists for exactly this (entry
|
|
189
|
+
renames to underscore spellings, every reference rewritten, ambiguous
|
|
190
|
+
renames refused, accepted under no_worse). A structural-flag pass over the
|
|
191
|
+
~287-book population is the honest fix; the spaces are convention, not
|
|
192
|
+
loss.
|
|
193
|
+
|
|
194
|
+
### RSC-007: TWO populations, both legacy-EPUB2 scars (wave 1, agent D)
|
|
195
|
+
|
|
196
|
+
300-book sample: the xpgt hypothesis (H1) is **real but not the whole
|
|
197
|
+
story**.
|
|
198
|
+
|
|
199
|
+
1. **The xpgt cluster**: 45 of 300 books reference `xpgt` (~46 refs per
|
|
200
|
+
referencing book), but 39 of them CARRY the template; only 6 of 300
|
|
201
|
+
reference it without carrying it, projecting ~141 of the 692 RSC-007
|
|
202
|
+
books. High mass though: 266 of 450 unresolved-reference occurrences in
|
|
203
|
+
the sample (59%) are xpgt (a handful of books at ~40-53 dangling refs
|
|
204
|
+
each, e.g. Global Crisis: 53, all `text/partNNNN.html ->
|
|
205
|
+
page-template.xpgt`). The reference is one shape everywhere:
|
|
206
|
+
`<link href="../Misc/page-template.xpgt" rel="stylesheet"
|
|
207
|
+
type="application/vnd.adobe-page-template+xml"/>` repeated in every
|
|
208
|
+
content doc, with converter-layout variants of the target path
|
|
209
|
+
(`Misc/`, `../styles/`, `ops/styles/`, `ArrowofGod/styles/`,
|
|
210
|
+
`page_template.xpgt`). This is the Adobe-era page template that a
|
|
211
|
+
converter dropped from the archive while leaving the per-document
|
|
212
|
+
links: **content-neutral loss** (the file never existed in the
|
|
213
|
+
archive, so no styling was ever applied), and `--prune-missing-resources`
|
|
214
|
+
is the honest repair for it: dropping dead `<link>` tags strictly
|
|
215
|
+
decreases the error count, normal gate.
|
|
216
|
+
2. **The diffuse loss tail**: 50 of 300 books have at least one unresolved
|
|
217
|
+
internal reference (ceiling projection ~1,175 books, ~10.5k
|
|
218
|
+
occurrences; the counter did not skip `kindle:` URIs or fragment junk,
|
|
219
|
+
so treat as an upper bound). Median 2 per book: stripped image
|
|
220
|
+
sequences (`00012.jpg` x36 in one book), Calibre's own generated
|
|
221
|
+
stylesheets gone missing (`page_styles.css` x12, `stylesheet.css` x12),
|
|
222
|
+
missing covers (`cover.xhtml` x10), stray images/fonts, broken NCX
|
|
223
|
+
targets, malformed hrefs (concatenation artifacts like
|
|
224
|
+
`..._cop_r1.htmlop`). Case-by-case, not one flag.
|
|
225
|
+
|
|
226
|
+
Both populations are ~90-98% `package version="2.0"`: the damage was baked
|
|
227
|
+
in by the legacy conversion chain, and the modern EPUB3 portion of the
|
|
228
|
+
library is structurally clean. Control: How Music Works (9691) belongs to
|
|
229
|
+
neither class (zero space names, zero xpgt, zero missing hrefs), so the
|
|
230
|
+
scars are pipeline-specific, not library-wide.
|
|
231
|
+
|
|
232
|
+
Projected reconciliation of the census's 692 RSC-007 books: ~141 xpgt
|
|
233
|
+
books (high per-book mass) + the diffuse tail reproduces the total within
|
|
234
|
+
sampling error.
|
|
235
|
+
|
|
236
|
+
### RSC-005: a narrow zoo behind a huge count (wave 1, agent A)
|
|
237
|
+
|
|
238
|
+
20-book sample, epubcheck text output, reason texts normalized into
|
|
239
|
+
families. Total 129,312 RSC-005 **text occurrences** in 16 of 20 books, but
|
|
240
|
+
the sample is extremely skewed: four books carry 99.2% (Modern Japan
|
|
241
|
+
76,859, Against the Gods 31,068, Rest Is Noise 12,658, Theory of Poker
|
|
242
|
+
7,931). Sixteen of the 20 books are clean or near-clean on this code
|
|
243
|
+
(median ~28 per book). In aggregated-message terms (the census scale) this
|
|
244
|
+
sample's dominant families are exactly the ones that produce one entry per
|
|
245
|
+
distinct defect text.
|
|
246
|
+
|
|
247
|
+
| family | text occ | books | flag judgment |
|
|
248
|
+
|---|---|---|---|
|
|
249
|
+
| Duplicate id, 99.97% inside .svg files (`Duplicate "glNNNN"` gradient/filter ids, thousands per file) | 115,861 | 4 | `--fix-svg-dup-ids` (high confidence; built for this) |
|
|
250
|
+
| element not allowed here (content model; 97% is `span` directly in body under XHTML 1.1; also br/a/div/p/tbody; 2 OPF oddities: ns0:timestamp, ns1:title_sort) | 9,210 | 7 | `--reserialize` (medium; html5lib reparse makes body-level inline legal in HTML5). The OPF oddities: no flag |
|
|
251
|
+
| element incomplete (blockquote lacking a block child under XHTML 1.1) | 3,639 | 5 | `--reserialize` (same reasoning) |
|
|
252
|
+
| text not allowed here (bare text as body child) | 256 | 4 | `--reserialize` (medium-high) |
|
|
253
|
+
| id values with colons (`X:Y`) | 227 | 3 | `--fix-id-colons` (high; message names it verbatim) |
|
|
254
|
+
| attribute not allowed (deprecated: clear 51, charset 25, link 10, vlink 10, valign 5, border 1; OPF page-map 2; EPUB3 `properties` on an EPUB2 package 1) | 106 | 7 | `--strip-bad-attrs` (content docs), `--strip-epub3-attrs` (the properties hit) |
|
|
255
|
+
| invalid lang value | 5 | 1 | no honest flag (rewrite, not strip) |
|
|
256
|
+
| playOrder conflict (same target, two values) | 5 | 2 | no dedicated flag |
|
|
257
|
+
| pageList missing class; OPF missing nav property | 3 | 2 | no dedicated flag |
|
|
258
|
+
|
|
259
|
+
Takeaways: the RSC-005 mass is **duplicate SVG gradient ids in a handful
|
|
260
|
+
of conversion-heavy books**, and `--fix-svg-dup-ids` is the existing,
|
|
261
|
+
purpose-built repair. The XHTML 1.1 content-model families (span-in-body,
|
|
262
|
+
incomplete blockquote, bare body text) are the legacy-doctype shape that
|
|
263
|
+
`--reserialize` neutralizes by re-parsing as HTML5. Agent A's caveat: a
|
|
264
|
+
single heavily-SVG history book produced 60% of the sample volume, so the
|
|
265
|
+
library-wide family distribution is likely flatter; a stratified re-sample
|
|
266
|
+
off the v0.53.1 artifact's books array is queued as wave 2.
|
|
267
|
+
|
|
268
|
+
#### RSC-005 resample (wave 2): the typical book is epub3-semantics-on-legacy-doctype, not SVG
|
|
269
|
+
|
|
270
|
+
The 30-book stratified resample (Random(7), census pool minus the known
|
|
271
|
+
heavyweights) overturns the first sample's model:
|
|
272
|
+
|
|
273
|
+
- **The dominant family is EPUB3 semantics rejected by legacy DTDs**:
|
|
274
|
+
"attribute not allowed" = 10,403 of 12,364 resample occurrences across 18
|
|
275
|
+
books, led by `role` (6,645), `aria-label` (1,737), `epub:type` (1,453),
|
|
276
|
+
`aria-hidden` (239). One converter habit: modern semantic attributes on
|
|
277
|
+
HTML4/XHTML1 doctypes.
|
|
278
|
+
- The first sample's SVG duplicate-id family is **entirely absent** here
|
|
279
|
+
(0 of 12,364): duplicate ids are a per-book quirk of a handful of
|
|
280
|
+
SVG-annotated titles, contributing essentially nothing to the other
|
|
281
|
+
~3,316 RSC-005 books.
|
|
282
|
+
- Mass concentration again, different books: five long-prose books (Some
|
|
283
|
+
Desperate Glory, House of Sky and Breath, Storming Heaven, The City and
|
|
284
|
+
Its Uncertain Walls, The Tiger) carry 91.5% of resample occurrences;
|
|
285
|
+
median 44; 13 of 30 books under 15. Zero of 30 books read clean, so the
|
|
286
|
+
census's RSC-005 book set is real.
|
|
287
|
+
- New metadata-level families the first sample missed: `@refines` missing
|
|
288
|
+
target id, invalid epub3 role value, href-not-a-URI,
|
|
289
|
+
Property-not-refining-creator, navList/pageList missing required
|
|
290
|
+
attribute. No flags.
|
|
291
|
+
|
|
292
|
+
**Flag gap (verified in source)**: `--strip-epub3-attrs`' fixed set is
|
|
293
|
+
exactly three attributes (`page-progression-direction`, `epub:type`,
|
|
294
|
+
`aria-label`). It covers aria-label and epub:type of the dominant family
|
|
295
|
+
but **not `role` or `aria-hidden`**, and its gate keys on the OPF package
|
|
296
|
+
version while epubcheck's rejection here is DOCTYPE-driven (an EPUB3
|
|
297
|
+
package carrying XHTML 1.1 doctypes would show the same findings yet be
|
|
298
|
+
skipped by the version gate). Extending the set and re-examining the gate
|
|
299
|
+
condition is the fix candidate for the family.
|
|
300
|
+
|
|
301
|
+
**Census-undercount confirmed systematically**: the artifact's per-book
|
|
302
|
+
error counts are unique-message-text counts; verified exactly on all 30
|
|
303
|
+
books, with undercount up to ~600x (one book: 5,946 occurrences collapse
|
|
304
|
+
to 10 census entries). Any repair prioritization must re-measure
|
|
305
|
+
occurrences directly rather than sort by the census field; the schema
|
|
306
|
+
revision (rename + a true occurrences count) is now load-bearing, not
|
|
307
|
+
cosmetic.
|
|
308
|
+
|
|
309
|
+
### THE 313 REJECTS: SOLVED. Two transform bugs, un-masking killed (wave 1, agent C)
|
|
310
|
+
|
|
311
|
+
The 12-book message-diff surgery (before/after epubcheck on repairs
|
|
312
|
+
regenerated through the same code path the gate measures) kills the
|
|
313
|
+
un-masking hypothesis outright: **zero books had a masking construct** (no
|
|
314
|
+
prolog errors, no fatals, no prior playOrder findings), and every +delta is
|
|
315
|
+
a violation the fix itself wrote into previously clean bytes, verified at
|
|
316
|
+
byte level at identical coordinates. The gate's refusals were all correct:
|
|
317
|
+
313 saves, not 313 misses.
|
|
318
|
+
|
|
319
|
+
**Bug 1: `fix_ncx_playorder` breaks contract-valid NCXs (199 of the 313
|
|
320
|
+
rejects).** The transform re-sequences `playOrder` across `<navPoint>` start
|
|
321
|
+
tags to document-order 1..N unconditionally. epubcheck's actual playOrder
|
|
322
|
+
contract: navPoints whose `<content src>` point at the same target must
|
|
323
|
+
SHARE one playOrder value; distinct targets need distinct values; no gaps,
|
|
324
|
+
and `pageTarget`/`navTarget` elements participate in the same sequence.
|
|
325
|
+
Real TOCs satisfy this with duplicate-target sharing (steel-breeze: two
|
|
326
|
+
navPoints at the same split_000.html, both `playOrder="1"`, zero findings
|
|
327
|
+
before, renumbered to 1/2 with a new finding at exactly those lines) and
|
|
328
|
+
with pageTargets interleaved (chronomancy: navPoints rewritten to 1,2,3
|
|
329
|
+
while the untouched `pageTarget ... playOrder="442"` at line 3034 now reads
|
|
330
|
+
as a sequence gap; mushroom-blues: 440 pageTargets, 0 -> 56 inconsistent
|
|
331
|
+
targets after the rewrite). Fix-mix evidence: 199 reject records carry
|
|
332
|
+
`fix_ncx_playorder` alone.
|
|
333
|
+
|
|
334
|
+
**Bug 2: `strip_prolog_junk` amputates the trailing `*/` of Calibre's CDATA
|
|
335
|
+
style wrappers (~110 of the 313).** `_PROTECTED_RE` matches
|
|
336
|
+
`<![CDATA[.*?]]>` non-greedily, so inside `/*<![CDATA[*/ ... /*]]>*/` the
|
|
337
|
+
protected span ends at the first `]]>` (inside the closing `/*]]>*/`),
|
|
338
|
+
leaving the trailing `*/` as the next outside segment. strip_prolog_junk
|
|
339
|
+
amputates everything before that segment's first `<`, eating `*/` plus
|
|
340
|
+
whitespace before `</style>` (the whitespace-only guard cannot catch `*`),
|
|
341
|
+
truncating the CSS comment, and epubcheck reports one CSS-008 "Premature
|
|
342
|
+
end of file" per broken file. Every fired instance in the sample is a
|
|
343
|
+
PHANTOM fix on a file with no prolog junk at all: fired counts equal
|
|
344
|
+
wrapper-file counts exactly (dark-star 18/18, sisyphus 6/6, calcutta 22/22,
|
|
345
|
+
moons-jupiter 21/21). At gate scale (aggregated entries) each book reads
|
|
346
|
+
+1.
|
|
347
|
+
|
|
348
|
+
**fix_named_entities: no breakage observed** (fired 3 times, all in one
|
|
349
|
+
book alongside the two buggy fixes; its mechanism cannot produce either
|
|
350
|
+
failure class; the same book's NCX-001 uid mismatch was genuinely repaired
|
|
351
|
+
by the uid sync, the one message-set loss in 12 books).
|
|
352
|
+
|
|
353
|
+
Consequences and scale note: both bugs sit in the DEFAULT pass (the NCX
|
|
354
|
+
pipeline is always-on; strip_prolog_junk is one of the five core
|
|
355
|
+
well-formedness fixes), so every library run attempts them; the gate is the
|
|
356
|
+
only thing that stood between the 199 books and a rewritten-but-worse NCX.
|
|
357
|
+
The aggregated-count scale also explains the tiny deltas: one bug hitting
|
|
358
|
+
22 files reads as exactly +1. Evidence preserved in
|
|
359
|
+
`testing_facility/error-surgery-2026-10-24/rejects/` (before/after epubcheck
|
|
360
|
+
outputs, probe records, and five specimen books); an independent
|
|
361
|
+
claim-verification pass is in flight before any code change is proposed.
|
|
362
|
+
|
|
363
|
+
Candidate repairs (to be scoped after verification): fix_ncx_playorder
|
|
364
|
+
should only fire when the NCX actually violates the playOrder contract, and
|
|
365
|
+
its renumbering must share values across same-target navPoints and include
|
|
366
|
+
pageTarget/navTarget in the sequence; strip_prolog_junk needs the
|
|
367
|
+
protection span to cover the wrapper closer (`/*]]>*/` as a unit) or an
|
|
368
|
+
orphan-`*/` guard before the whitespace test.
|
|
369
|
+
|
|
370
|
+
#### Independent verification: 4/4 claims confirmed, zero refuted
|
|
371
|
+
|
|
372
|
+
The claim-verifier re-proved both bugs from scratch with minimal synthetic
|
|
373
|
+
reproducers and re-derived every count from the dry-run JSON:
|
|
374
|
+
|
|
375
|
+
- **Claim 1 CONFIRMED**: a minimal EPUB whose NCX shares `playOrder="1"`
|
|
376
|
+
across two navPoints on the same target validates clean; the transform
|
|
377
|
+
in isolation fires 2 (a phantom fix on a valid file), splits the shared
|
|
378
|
+
value to 1/2/3, and the repacked book fails with the exact claimed
|
|
379
|
+
finding, twice. Exactly 199 reject records carry
|
|
380
|
+
`{fix_ncx_playorder}` alone (deltas +1 x121, +2 x60, +3 x18); two of the
|
|
381
|
+
313 rejects were 0f/0e fully clean books damaged by phantom fixes.
|
|
382
|
+
- **Claim 2 CONFIRMED**: isolation on a Calibre-wrapped inline style shows
|
|
383
|
+
the removed bytes are exactly `*/` (the style close becomes
|
|
384
|
+
`/*]]></style>`), the fired count is 1 on a file with no prolog junk,
|
|
385
|
+
and the repacked book fails with CSS-008 "Premature end of file".
|
|
386
|
+
111 reject records contain strip_prolog_junk (73 solo + 29 with
|
|
387
|
+
fix_named_entities + 7 + 2 mixed).
|
|
388
|
+
- **Claim 3 CONFIRMED**: a 3-stylesheet demo shows 3 text occurrences
|
|
389
|
+
collapsing to 1 JSON entry; vol3 reproduces 20,307 text vs 7,030 JSON
|
|
390
|
+
exactly; the gate code reads `checker.nError` (validate.py).
|
|
391
|
+
- **Claim 4 CONFIRMED and strengthened**: fix_named_entities fired SOLO in
|
|
392
|
+
93 records with zero rejections (14 accept at -1, 78 equal, 1 partial):
|
|
393
|
+
when alone it repairs or is neutral, never breaks. All 34 mixed records
|
|
394
|
+
are attributable to their companions.
|
|
395
|
+
- Verifier's aside (a third sequencing blind spot): a navPoint with no
|
|
396
|
+
playOrder attribute neither consumes nor receives a sequence number.
|
|
397
|
+
- Bookkeeping note: the facility's rejects/ dir holds 6 of the 12 sampled
|
|
398
|
+
books (the /tmp forensics set holds all 12); widen the facility set
|
|
399
|
+
before any fix session that wants the full spread.
|
|
400
|
+
|
|
401
|
+
#### The benefit ratio that decides the fix design
|
|
402
|
+
|
|
403
|
+
Across the entire dry run, fix_ncx_playorder fired on 579 books:
|
|
404
|
+
|
|
405
|
+
| outcome | books |
|
|
406
|
+
|---|---|
|
|
407
|
+
| equal (fired, zero count change: a no-op rewrite) | 347 (60%) |
|
|
408
|
+
| reject with positive delta (actively made worse) | 211 (36%) |
|
|
409
|
+
| accept with negative delta (the legitimate use) | **17 (3%)** |
|
|
410
|
+
| partial / error | 4 |
|
|
411
|
+
|
|
412
|
+
The transform helps 3% of the books it fires on and harms 36%. The
|
|
413
|
+
contract-gating repair (fire only when the NCX actually violates the
|
|
414
|
+
playOrder contract) turns the 347 no-ops and 211 harms into no-ops and
|
|
415
|
+
preserves the 17 real fixes.
|
|
416
|
+
|
|
417
|
+
### Disposition probes: all three proposed flags CONFIRMED live (wave 2)
|
|
418
|
+
|
|
419
|
+
Each disposition was probed on a real library book (specimens in the
|
|
420
|
+
facility; gate records and before/after epubcheck transcripts preserved):
|
|
421
|
+
|
|
422
|
+
| class | book | flag | target delta | gate |
|
|
423
|
+
|---|---|---|---|---|
|
|
424
|
+
| RSC-005 svg dup ids | Making of Modern Japan | `--fix-svg-dup-ids` | 76,859 occurrences -> 0; **fully clean epubcheck** | accept, 104 svg entries touched, 71,711 ids fixed |
|
|
425
|
+
| RSC-007 xpgt dead links | Global Crisis | `--prune-missing-resources` | 53 RSC-007 -> 0; sole residual is an out-of-class RSC-020 mailto-space error present in the baseline | accept, dead_links_pruned=53 |
|
|
426
|
+
| PKG-010 spaces | Stars My Destination | `--encode-url-spaces` | 20 PKG-010 -> 0 AND 21 collateral RSC-020 -> 0; **fully clean epubcheck** | accept, 20 renames + 21 refs rewritten, zero rename-caused misses |
|
|
427
|
+
|
|
428
|
+
Two cross-cutting notes: the encode-url-spaces probe cleared the book's
|
|
429
|
+
entire RSC-020 population too (those were spaces-in-URL findings, the same
|
|
430
|
+
defect on the URL axis), so the two classes overlap on that shape; and
|
|
431
|
+
bindery's internal before-count for Modern Japan read 788 (aggregated
|
|
432
|
+
entries) against epubcheck's 76,859 (text locations) — the aggregation
|
|
433
|
+
scale again, consistent end to end.
|
|
434
|
+
|
|
435
|
+
### RSC-020: SOLVED. The PKG-010 twin plus a junk-link tail (wave 2)
|
|
436
|
+
|
|
437
|
+
15-book weighted sample off the artifact's books array (5
|
|
438
|
+
fingerprint-positive + 10 arbitrary; 178 raw lines, 127 unique URLs):
|
|
439
|
+
|
|
440
|
+
| family | raw lines | verdict |
|
|
441
|
+
|---|---|---|
|
|
442
|
+
| 1. space-in-URL, internal relative href, target exists | 133 (75%) | invalid-but-harmless naming; **the PKG-010 twin** |
|
|
443
|
+
| 2. escaped-markup/garbage href (`<i>` swallowed into an href, Windows path) | 20 | genuinely broken (missing internal targets) |
|
|
444
|
+
| 3. space-in-URL, external http(s) (line-wrap artifacts baked into hrefs) | 12 | genuinely broken addresses even after encoding |
|
|
445
|
+
| 4. %20 spliced mid-host | 7 | genuinely broken; encoding cannot fix (epubcheck percent-decodes then rejects the space) |
|
|
446
|
+
| 5. mailto-with-space | 2 | genuinely broken (unsendable) |
|
|
447
|
+
| 6. literal space in host | 1 | genuinely broken; still rejected if encoded |
|
|
448
|
+
| 7. illegal bracket in relative filename, target exists | 2 | harmless naming (needs %5B%5D or a rename, never a space flag) |
|
|
449
|
+
| 8. malformed host (empty domain label) | 1 | genuinely broken garbage |
|
|
450
|
+
|
|
451
|
+
Population overlap: **287 of the 353 RSC-020 books (81.3%) carry the
|
|
452
|
+
PKG-010 space-entry fingerprint**, and in every fingerprint-positive book
|
|
453
|
+
the counts are in strict three-way identity (PKG-010 count = zip
|
|
454
|
+
space-entry count = flagged internal space-URL count): one root cause, two
|
|
455
|
+
epubcheck layers. The `--encode-url-spaces` disposition therefore covers
|
|
456
|
+
family 1 wholesale (confirmed live on Stars My Destination, where all 21
|
|
457
|
+
RSC-020s cleared as collateral) and makes families 3 and 5 *parse* as
|
|
458
|
+
valid URLs while their targets stay dead (cosmetic repair at best). The
|
|
459
|
+
genuinely-broken families 2/4/6/8 live in the 66 fingerprint-negative
|
|
460
|
+
books, are near-zero user impact (junk links, mostly external), and have
|
|
461
|
+
no honest flag; leave them measured, not "repaired".
|
|
462
|
+
|
|
463
|
+
Counting-semantics proof from this lane: the artifact aggregates unique
|
|
464
|
+
flagged URLs per book (The Chill: 34 artifact vs 66 raw lines, each URL
|
|
465
|
+
flagged in both content.opf and toc.ncx), further confirming the
|
|
466
|
+
unique-text scale of every census number.
|
|
467
|
+
|
|
468
|
+
### --reserialize: FALSIFIED for the content-model family, and the bug bites again (wave 2)
|
|
469
|
+
|
|
470
|
+
The Dark Star probe (the span-in-body exemplar) came back with two
|
|
471
|
+
structural findings:
|
|
472
|
+
|
|
473
|
+
1. **The gate rejected the probe, and the regression is the prolog bug
|
|
474
|
+
again**: the fixes dict carries only `strip_prolog_junk: 18`, the
|
|
475
|
+
refusal is `0f/7e -> 0f/8e` (one aggregated CSS-008 entry), and a
|
|
476
|
+
control run without `--reserialize` produces a byte-identical reject
|
|
477
|
+
record. The core pass never let the opt-in stage run. The 18 CSS-008s
|
|
478
|
+
are exactly the class bindery's own `--fix-cdata-terminator` exists to
|
|
479
|
+
repair: the core pass manufactures work for an opt-in.
|
|
480
|
+
2. **Even gate-free, reserialize is inert here**: its html5lib path
|
|
481
|
+
triggers only on XML ParseError ("runs only on documents that are not
|
|
482
|
+
already well-formed"), and the content-model family is
|
|
483
|
+
well-formed-but-schema-invalid. The no-validate hypothetical ran the
|
|
484
|
+
rebuild trigger zero times; all 273 RSC-005 content-model occurrences
|
|
485
|
+
survived; text preservation confirmed only that the strip damage lost
|
|
486
|
+
no prose.
|
|
487
|
+
|
|
488
|
+
The medium-confidence research claim is falsified for this exemplar: the
|
|
489
|
+
span-in-body / incomplete-blockquote / bare-body-text families have **no
|
|
490
|
+
working repair today**. Addressing them would mean a new opt-in that wraps
|
|
491
|
+
loose block content (or an explicit second reserialize trigger for
|
|
492
|
+
well-formed docs with content-model errors, contradicting its current
|
|
493
|
+
"good files are untouched" contract). Until then they stay measured, not
|
|
494
|
+
repaired.
|
|
495
|
+
|
|
496
|
+
## Closing recommendations (2026-10-24 surgery)
|
|
497
|
+
|
|
498
|
+
1. **Fix the two core-pass bugs first** (both in the always-on pass, both
|
|
499
|
+
gate-proven, independently verified with synthetic reproducers):
|
|
500
|
+
- `fix_ncx_playorder`: fire only when the NCX actually violates the
|
|
501
|
+
playOrder contract; share values across same-target navPoints;
|
|
502
|
+
include pageTarget/navTarget in the sequence; handle navPoints with
|
|
503
|
+
no playOrder attribute (the verifier's third blind spot). Benefit
|
|
504
|
+
ratio today: 17 helped / 211 harmed / 347 no-ops out of 579 firings.
|
|
505
|
+
- `strip_prolog_junk`: treat the Calibre wrapper `/*<![CDATA[*/ ...
|
|
506
|
+
/*]]>*/` as one protected unit (or guard the orphan `*/` before the
|
|
507
|
+
whitespace test). Today it phantom-fires on files with no prolog junk
|
|
508
|
+
and manufactures CSS-008 x N.
|
|
509
|
+
Re-run the whole-library dry run after the fix: the 313 rejects should
|
|
510
|
+
collapse to near-zero, and the 347/equal no-op rewrites should stop
|
|
511
|
+
firing.
|
|
512
|
+
2. **Then the validated structural pass** (one reviewed apply run, same
|
|
513
|
+
backup pipeline as stage 2/2b): `--fix-svg-dup-ids` +
|
|
514
|
+
`--prune-missing-resources` + `--encode-url-spaces`. All three are
|
|
515
|
+
gate-proven to fully clear their classes on real specimens; the run
|
|
516
|
+
also clears RSC-020 family 1 as collateral. Ordering matters: the
|
|
517
|
+
structural pass must wait for the bug fixes because a book carrying
|
|
518
|
+
both the fake-CDATA idiom and a target defect would be refused
|
|
519
|
+
wholesale by the gate (the buggy core fix poisons the bundle).
|
|
520
|
+
3. **Extend `--strip-epub3-attrs`** (`role`, `aria-hidden`) and re-examine
|
|
521
|
+
its package-version gate against the doctype-driven rejection, before
|
|
522
|
+
any pass aimed at the dominant RSC-005 family. Check the target
|
|
523
|
+
population's package versions first.
|
|
524
|
+
4. **Schema revision for observe (v2)**: rename `codes.occurrences` ->
|
|
525
|
+
`codes.messages` (aggregated entries), add a true occurrences count
|
|
526
|
+
(locations), state the scale in the artifact and docs. Now load-bearing:
|
|
527
|
+
prioritization off per-book counts can mis-rank by ~600x.
|
|
528
|
+
5. **No-repair list (leave measured)**: the Feynman equation scar
|
|
529
|
+
(re-source), the XHTML 1.1 content-model families (no working repair;
|
|
530
|
+
reserialize falsified), RSC-020 junk-link families 2/4/6/8, the
|
|
531
|
+
diffuse RSC-007 tail (case-by-case), the metadata-level RSC-005
|
|
532
|
+
families. These are the honest residue; the observatory tracks them.
|
|
533
|
+
6. **Candidate new flag (low priority)**: flat-repack re-point (the
|
|
534
|
+
Gotrek & Felix cover-image class: the target exists under its flat
|
|
535
|
+
name; a basename-elsewhere re-point would fix 1-9-occurrence escapes).
|
|
536
|
+
|
|
537
|
+
Evidence: specimens and probe records under
|
|
538
|
+
`testing_facility/error-surgery-2026-10-24/` (rsc026/ rsc005/ rejects/
|
|
539
|
+
pkg010/), census artifacts under `~/docs/Calibre Library.backups/observe/`,
|
|
540
|
+
the dry-run JSON under the library's `.claude/`. All measurement was
|
|
541
|
+
read-only; the library was never written during this research.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bindery-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.54.0
|
|
4
4
|
Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
|
|
5
5
|
Project-URL: Homepage, https://github.com/VirInvictus/bindery-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/VirInvictus/bindery-cli
|
|
@@ -1,5 +1,70 @@
|
|
|
1
1
|
# bindery-cli Patch Notes
|
|
2
2
|
|
|
3
|
+
## v0.54.0 (2026-10-24)
|
|
4
|
+
|
|
5
|
+
### The error-code surgery fixes: the core pass stopped breaking valid books
|
|
6
|
+
|
|
7
|
+
The 2026-10-24 whole-library surgery (the record lives in
|
|
8
|
+
`OBSERVE-RESEARCH.md`, specimens in the gitignored testing facility)
|
|
9
|
+
proved that the tool's two worst whole-library behaviors, 313 gate
|
|
10
|
+
rejections and a wall of no-change rewrites, were both self-inflicted.
|
|
11
|
+
Two always-on core transforms carried bugs; the epubcheck gate correctly
|
|
12
|
+
refused every damaged output, which is why no library book was ever hurt.
|
|
13
|
+
|
|
14
|
+
**fix_ncx_playorder rewrote valid TOCs into violations.** The old form
|
|
15
|
+
renumbered every `<navPoint>` playOrder to document-order 1..N
|
|
16
|
+
unconditionally. epubcheck's actual contract is subtler: navPoints whose
|
|
17
|
+
`<content src>` point at the same target must SHARE one playOrder value,
|
|
18
|
+
distinct targets need distinct values, and pageTarget/navTarget elements
|
|
19
|
+
participate in the same contiguous sequence. Real TOCs satisfy this with
|
|
20
|
+
duplicate-target sharing and interleaved pageTargets, so the renumbering
|
|
21
|
+
manufactured RSC-005 playOrder findings on books whose NCX had zero
|
|
22
|
+
findings before. Whole-library tally: 579 firings, 17 real fixes, 211
|
|
23
|
+
gate-rejected regressions, 347 no-op rewrites. The transform now parses
|
|
24
|
+
the contract (element kind, playOrder value, content target in document
|
|
25
|
+
order), returns unchanged when the NCX already satisfies it, and on a
|
|
26
|
+
violation re-sequences by first-appearance target order, rewriting only
|
|
27
|
+
the attributes whose value changes. Verified against a minimal synthetic
|
|
28
|
+
EPUB: a shared-target NCX that validates clean goes in and comes out
|
|
29
|
+
untouched; the genuinely broken shapes (all-1 NCX, split shared values,
|
|
30
|
+
sequence gaps) still repair.
|
|
31
|
+
|
|
32
|
+
**strip_prolog_junk amputated the trailing `*/` of Calibre's CDATA style
|
|
33
|
+
wrappers.** The protection map's CDATA alternative ends at the first
|
|
34
|
+
`]]>`, which inside Calibre's `/*<![CDATA[*/ ... /*]]>*/` idiom sits
|
|
35
|
+
before the wrapper's own closing `*/`. That orphan landed in the outside
|
|
36
|
+
segment, the prolog strip ate it as "stray bytes before the first `<"`
|
|
37
|
+
(the whitespace guard cannot catch `*`), and every wrapped stylesheet
|
|
38
|
+
came back with a truncated comment: one CSS-008 "Premature end of file"
|
|
39
|
+
per file, a phantom fix whose fired count exactly equaled the wrapper
|
|
40
|
+
count (18/18, 22/22, 6/6 in the samples). The protection map now carries
|
|
41
|
+
the Calibre wrapper as its own first alternative, so the idiom is one
|
|
42
|
+
protected unit; a genuinely truncated closer (no trailing `*/`) stays
|
|
43
|
+
outside the span, where `--fix-cdata-terminator` can still reach it.
|
|
44
|
+
|
|
45
|
+
**fix_named_entities is exonerated** by the same surgery: it fired solo
|
|
46
|
+
in 93 whole-library records with zero rejections (14 real fixes, 78
|
|
47
|
+
count-neutral). No change.
|
|
48
|
+
|
|
49
|
+
Verification: 9-agent research pass with an independent claim-verification
|
|
50
|
+
lane (4/4 claims confirmed, minimal synthetic reproducers for both bugs);
|
|
51
|
+
730 tests green (the playorder suite now pins the contract semantics,
|
|
52
|
+
including the shared-target regression test); a whole-library dry run
|
|
53
|
+
after the fix confirms the rejection collapse.
|
|
54
|
+
|
|
55
|
+
## v0.53.1 (2026-10-24)
|
|
56
|
+
|
|
57
|
+
### The .observe artifact now carries its per-book census rows
|
|
58
|
+
|
|
59
|
+
v0.53.0's spec, README, and patchnotes promised per-book census rows in the
|
|
60
|
+
artifact ("any later question can be answered without re-validating"), and
|
|
61
|
+
the first real run proved the promise hollow: the writer composed every
|
|
62
|
+
aggregate but never the `books` array. The rows are written now, one record
|
|
63
|
+
per book, clean ones included: path relative to the root, counts, the
|
|
64
|
+
error-code list, and the unreadable flag. The schema is unchanged
|
|
65
|
+
(`bindery-observe-v1`: the field set grew; no existing consumer reads the
|
|
66
|
+
array), and the test pins the rows.
|
|
67
|
+
|
|
3
68
|
## v0.53.0 (2026-10-24)
|
|
4
69
|
|
|
5
70
|
### `bindery observe`: the whole-library census, one artifact per run
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "bindery-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.54.0"
|
|
8
8
|
description = "Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
classifiers = [
|
|
@@ -46,7 +46,8 @@ The plugin vendors the core and never gates (the epubcheck latency ruling).
|
|
|
46
46
|
| 0.50 | The contamination analyzer (issue #8): foreign-title template stamps, fan-bundle ad signatures, review-paste front matter; advisory by contract, archive-wide corpus (`Book.extra_docs`, capped), and the `_dir_dispatch` map that fixes the v0.44.0 cover/tocdrift ocr misroute in directory/single mode | v0.50.0 |
|
|
47
47
|
| 0.51 | The unbound-prefix repair (issue #9): `--fix-unbound-prefix` binds the Calibre embed scar's undeclared attribute prefix inline (evidence-based URI, scope-tracked, refuse over guess), plus the masked-ncx-gain rule that stops a real dtb:uid sync reading as no measurable gain under a fatal package | v0.51.0 |
|
|
48
48
|
| 0.52 | The spine guard (issue #10): the manifest prune reads the layout its hrefs are written in and derives its spine protection from the text it prunes (the raw-space wave's spine collapse), and `process_book` vetoes any candidate whose spine (itemref -> item -> archive) dangles more than the source's, above the counts and every promotion | v0.52.0 |
|
|
49
|
-
| 0.53 | `bindery observe` (the observatory): the whole-library census in one warm-JVM pass (counts and error codes together, sentinel-backed so no book is dropped), tabulation (severity histogram, code profile, worst offenders), the gated repair preview composed in-process, the versioned `.observe` artifact with `--against` trend diffing and the keep-root-clean output guard; the library and audit walks skip `.caltrash` | v0.53.0 |
|
|
49
|
+
| 0.53 | `bindery observe` (the observatory): the whole-library census in one warm-JVM pass (counts and error codes together, sentinel-backed so no book is dropped), tabulation (severity histogram, code profile, worst offenders), the gated repair preview composed in-process, the versioned `.observe` artifact with `--against` trend diffing and the keep-root-clean output guard; the library and audit walks skip `.caltrash` | v0.53.0-v0.53.1 |
|
|
50
|
+
| 0.54 | The surgery fixes: `fix_ncx_playorder` is contract-gated (epubcheck's playOrder contract: same-target sharing, distinct-target distinctness, contiguous 1..K across navTarget/pageTarget; the old unconditional 1..N renumbering rewrote valid TOCs into violations, 211 of 579 firings gate-rejected vs 17 real fixes) and `_PROTECTED_RE` protects the Calibre `/*<![CDATA[*/ ... /*]]>*/` wrapper as one unit (the bare CDATA span exposed the trailing `*/` to strip_prolog_junk's amputation, 111 phantom-fix rejections); verified by the 2026-10-24 error-code surgery (OBSERVE-RESEARCH.md) | v0.54.0 |
|
|
50
51
|
|
|
51
52
|
## Open work
|
|
52
53
|
|
|
@@ -46,7 +46,12 @@ Applied to content documents (`.xhtml`, `.html`, `.htm`, `.xml`), in order:
|
|
|
46
46
|
a declaration must be the very first thing in the document; before a DOCTYPE or the
|
|
47
47
|
root element it is legal prolog whitespace and is left in place. (Stripping it
|
|
48
48
|
anyway counted a fix on an undamaged document, which forced the archive rewrite's
|
|
49
|
-
lossy `decode("utf-8", "replace")` round-trip for no gain.)
|
|
49
|
+
lossy `decode("utf-8", "replace")` round-trip for no gain.) Calibre's inline-CSS
|
|
50
|
+
wrapper `/*<![CDATA[*/ ... /*]]>*/` is protected as one unit (v0.54.0): the bare
|
|
51
|
+
CDATA span ends inside the closing `/*]]>`, which left the trailing `*/` outside
|
|
52
|
+
the protection map, where the prolog strip amputated it -- a phantom fix
|
|
53
|
+
truncating the CSS comment (one CSS-008 "Premature end of file" per wrapped
|
|
54
|
+
file; 111 whole-library gate rejections).
|
|
50
55
|
2. **drop_duplicate_xmlns**: keep only the first `xmlns="..."` on the root `<html>`.
|
|
51
56
|
3. **escape_bare_amp**: replace a `&` that does not start a valid entity or character
|
|
52
57
|
reference with `&`.
|
|
@@ -63,9 +68,17 @@ Applied to content documents (`.xhtml`, `.html`, `.htm`, `.xml`), in order:
|
|
|
63
68
|
Applied to the NCX sidecar (`.ncx`): strip_prolog_junk, escape_bare_amp,
|
|
64
69
|
fix_named_entities, plus **dtb:uid sync** (rewriting the NCX's `dtb:uid`
|
|
65
70
|
meta to match the OPF's unique identifier when they drift, NCX-001) and
|
|
66
|
-
**fix_ncx_playorder** (resequencing `playOrder`
|
|
67
|
-
|
|
68
|
-
|
|
71
|
+
**fix_ncx_playorder** (resequencing `playOrder` values so the NCX
|
|
72
|
+
satisfies epubcheck's playOrder contract: elements whose `<content src>`
|
|
73
|
+
point at the same target share one value, distinct targets carry distinct
|
|
74
|
+
values, and the value set is contiguous 1..K across navPoint,
|
|
75
|
+
navTarget, and pageTarget elements together. Contract-gated since v0.54.0:
|
|
76
|
+
an NCX that already satisfies the contract is returned untouched -- the
|
|
77
|
+
previous unconditional 1..N renumbering rewrote valid TOCs into violations
|
|
78
|
+
(splitting duplicate-target sharing, colliding with pageTarget sequences;
|
|
79
|
+
211 of the 579 books it fired on gate-rejected, against 17 real fixes).
|
|
80
|
+
nav-label text is never touched; a navPoint with no playOrder attribute is
|
|
81
|
+
not part of the sequence).
|
|
69
82
|
With `--fix-ids`, ids in the NCX that are not valid XML names (digit-led, as when a
|
|
70
83
|
converter stamps navPoint ids from UUIDs; colon-bearing) are renamed with the same
|
|
71
84
|
`id_` scheme as OPF manifest ids. NCX ids are internal to the NCX (nothing in the
|
|
@@ -11,7 +11,7 @@ Calibre plugin built from this core.
|
|
|
11
11
|
Single source of truth for the version. pyproject.toml must match VERSION below.
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
|
-
VERSION = "0.
|
|
14
|
+
VERSION = "0.54.0"
|
|
15
15
|
__version__ = VERSION
|
|
16
16
|
|
|
17
17
|
from .epub import RepairReport, repair_epub # noqa: E402
|
|
@@ -480,6 +480,19 @@ def run_observe(args) -> int:
|
|
|
480
480
|
"root": str(root),
|
|
481
481
|
"census_seconds": round(census_s, 1),
|
|
482
482
|
**tab,
|
|
483
|
+
# The per-book census rows: the artifact answers later questions
|
|
484
|
+
# without re-validating, so every book lands here, clean or not.
|
|
485
|
+
"books": [
|
|
486
|
+
{
|
|
487
|
+
"path": os.path.relpath(r.path, root),
|
|
488
|
+
"fatals": r.fatals,
|
|
489
|
+
"errors": r.errors,
|
|
490
|
+
"warnings": r.warnings,
|
|
491
|
+
"codes": list(r.codes),
|
|
492
|
+
"unreadable": r.unreadable,
|
|
493
|
+
}
|
|
494
|
+
for r in records
|
|
495
|
+
],
|
|
483
496
|
}
|
|
484
497
|
|
|
485
498
|
if not getattr(args, "no_preview", False):
|
|
@@ -61,7 +61,18 @@ _EPUB_PREFIX_VAL_RE = re.compile(r"(?:^|\s)([A-Za-z_][\w.-]*)\s*:\s*\S+")
|
|
|
61
61
|
# (e.g. `&` in CDATA-wrapped CSS/JS renders as `&`; escaped, it renders as `&`).
|
|
62
62
|
# Splitting on this regex yields alternating outside/protected segments, so the
|
|
63
63
|
# markup transforms run only on the even (outside) indices.
|
|
64
|
-
|
|
64
|
+
#
|
|
65
|
+
# The first alternative is Calibre's inline-CSS wrapper idiom
|
|
66
|
+
# `/*<![CDATA[*/ ... /*]]>*/`: the bare CDATA alternative ends at the first `]]>`
|
|
67
|
+
# (inside the closing `/*]]>`), leaving the trailing `*/` as an outside segment,
|
|
68
|
+
# where strip_prolog_junk amputated it as "stray bytes before the first `<"` (a
|
|
69
|
+
# phantom fix that truncated the CSS comment and manufactured one CSS-008
|
|
70
|
+
# "Premature end of file" per wrapped file; the 2026-10-24 whole-library dry run
|
|
71
|
+
# gate-rejected 111 books on exactly this). The wrapper is protected as one unit.
|
|
72
|
+
_PROTECTED_RE = re.compile(
|
|
73
|
+
r"(/\*<!\[CDATA\[\*/.*?/\*\]\]>\*/|<!\[CDATA\[.*?\]\]>|<!--.*?-->)",
|
|
74
|
+
re.DOTALL,
|
|
75
|
+
)
|
|
65
76
|
|
|
66
77
|
|
|
67
78
|
def outside_protected_map(s: str, fn: Transform) -> tuple[str, int]:
|
|
@@ -323,38 +334,127 @@ def drop_duplicate_xmlns(s: str) -> tuple[str, int]:
|
|
|
323
334
|
return s, count
|
|
324
335
|
|
|
325
336
|
|
|
337
|
+
# The playOrder contract epubcheck enforces (the RSC-005 playOrder findings):
|
|
338
|
+
# elements whose <content src> point at the SAME target share one value,
|
|
339
|
+
# distinct targets carry distinct values, and the value set is contiguous
|
|
340
|
+
# 1..K across navPoint + navTarget + pageTarget elements together.
|
|
341
|
+
_PLAYORDER_EL_RE = re.compile(
|
|
342
|
+
r"<(?:navPoint|navTarget|pageTarget)\b[^>]*>", re.IGNORECASE
|
|
343
|
+
)
|
|
344
|
+
_PLAYORDER_ATTR_RE = re.compile(
|
|
345
|
+
r"\s+playOrder\s*=\s*(\"[^\"]*\"|'[^']*')", re.IGNORECASE
|
|
346
|
+
)
|
|
347
|
+
_CONTENT_SRC_RE = re.compile(
|
|
348
|
+
r"<content\b[^>]*?\ssrc\s*=\s*(\"[^\"]*\"|'[^']*')", re.IGNORECASE
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
|
|
326
352
|
@_outside_protected
|
|
327
353
|
def fix_ncx_playorder(s: str) -> tuple[str, int]:
|
|
328
|
-
"""Re-sequence `playOrder`
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
354
|
+
"""Re-sequence `playOrder` values so the NCX satisfies epubcheck's playOrder
|
|
355
|
+
contract, and touch nothing when it already does.
|
|
356
|
+
|
|
357
|
+
Contract-gated (the 2026-10-24 whole-library surgery): the previous form
|
|
358
|
+
renumbered every `<navPoint>` to document-order 1..N unconditionally, which
|
|
359
|
+
rewrote VALID TOCs into violations -- navPoints whose `<content src>` share
|
|
360
|
+
a target must share one playOrder (splitting them is a finding), pageTarget/
|
|
361
|
+
navTarget elements participate in the same sequence (ignoring them leaves
|
|
362
|
+
gaps), and the gate rejected 211 of the 579 books it fired on, against 17
|
|
363
|
+
real fixes. The resequencing here assigns each distinct target one value in
|
|
364
|
+
first-appearance order and rewrites only the attributes whose value changes.
|
|
365
|
+
|
|
366
|
+
Anchored to start tags of the three playOrder-bearing elements: the word
|
|
367
|
+
`playOrder` inside a nav label's text is content, not markup, and is never
|
|
368
|
+
touched. A navPoint with no playOrder attribute is not part of the sequence
|
|
369
|
+
(adding attributes is not this repair), and an element whose `<content src>`
|
|
370
|
+
cannot be located is treated as its own target. Known limit: the tag match
|
|
371
|
+
ends at the first `>`, so a `>` inside a quoted attribute value before the
|
|
372
|
+
playOrder attribute hides the element (the same single-class shape every
|
|
373
|
+
tag matcher here documents).
|
|
332
374
|
"""
|
|
333
|
-
|
|
334
|
-
|
|
375
|
+
elements: list[dict] = [] # (start, end, raw_value, target, order)
|
|
376
|
+
current: dict | None = None
|
|
377
|
+
for m in _PLAYORDER_EL_RE.finditer(s):
|
|
378
|
+
tag = m.group(0)
|
|
379
|
+
attr = _PLAYORDER_ATTR_RE.search(tag)
|
|
380
|
+
if attr is None:
|
|
381
|
+
continue
|
|
382
|
+
if current is not None:
|
|
383
|
+
elements.append(current)
|
|
384
|
+
current = {
|
|
385
|
+
"start": m.start(),
|
|
386
|
+
"end": m.end(),
|
|
387
|
+
"tag": tag,
|
|
388
|
+
"raw": attr.group(1),
|
|
389
|
+
"value": attr.group(1).strip("\"'"),
|
|
390
|
+
"target": None,
|
|
391
|
+
}
|
|
392
|
+
if current is not None:
|
|
393
|
+
elements.append(current)
|
|
394
|
+
if not elements:
|
|
395
|
+
return s, 0
|
|
335
396
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
if m
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
return
|
|
397
|
+
# Assign each element the src of the first <content> following its own start
|
|
398
|
+
# tag and before the next playOrder-bearing element starts (flat and
|
|
399
|
+
# parent-content-first nesting both assign correctly; children-content-first
|
|
400
|
+
# nesting is the documented limit).
|
|
401
|
+
for i, el in enumerate(elements):
|
|
402
|
+
next_start = elements[i + 1]["start"] if i + 1 < len(elements) else len(s)
|
|
403
|
+
m = _CONTENT_SRC_RE.search(s, el["end"], next_start)
|
|
404
|
+
if m is not None:
|
|
405
|
+
el["target"] = m.group(1).strip("\"'")
|
|
406
|
+
|
|
407
|
+
def key_of(el: dict) -> str:
|
|
408
|
+
# An element whose content src cannot be located is its own target
|
|
409
|
+
# (keyed by position so two such elements never merge).
|
|
410
|
+
return el["target"] if el["target"] is not None else f"\0{el['start']}"
|
|
411
|
+
|
|
412
|
+
groups: dict[str, list[dict]] = {}
|
|
413
|
+
for el in elements:
|
|
414
|
+
groups.setdefault(key_of(el), []).append(el)
|
|
415
|
+
|
|
416
|
+
# Contract check: every target group carries exactly one integer value,
|
|
417
|
+
# groups never share a value, and the value set is exactly 1..len(groups).
|
|
418
|
+
used: set[int] = set()
|
|
419
|
+
contract_ok = True
|
|
420
|
+
for members in groups.values():
|
|
421
|
+
vals: set[int] = set()
|
|
422
|
+
for el in members:
|
|
423
|
+
try:
|
|
424
|
+
vals.add(int(el["value"]))
|
|
425
|
+
except ValueError:
|
|
426
|
+
contract_ok = False
|
|
427
|
+
break
|
|
428
|
+
if not contract_ok or len(vals) != 1 or vals & used:
|
|
429
|
+
contract_ok = False
|
|
430
|
+
break
|
|
431
|
+
used |= vals
|
|
432
|
+
if contract_ok and used == set(range(1, len(groups) + 1)):
|
|
433
|
+
return s, 0
|
|
350
434
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
435
|
+
# Violating: re-sequence by first appearance of each target.
|
|
436
|
+
assigned: dict[str, int] = {}
|
|
437
|
+
out = []
|
|
438
|
+
last = 0
|
|
439
|
+
count = 0
|
|
440
|
+
for el in elements:
|
|
441
|
+
key = key_of(el)
|
|
442
|
+
if key not in assigned:
|
|
443
|
+
assigned[key] = len(assigned) + 1
|
|
444
|
+
new = str(assigned[key])
|
|
445
|
+
if el["value"] != new:
|
|
446
|
+
count += 1
|
|
447
|
+
attr = _PLAYORDER_ATTR_RE.search(el["tag"])
|
|
448
|
+
new_tag = (
|
|
449
|
+
el["tag"][: attr.start()]
|
|
450
|
+
+ f' playOrder="{new}"'
|
|
451
|
+
+ el["tag"][attr.end() :]
|
|
452
|
+
)
|
|
453
|
+
out.append(s[last : el["start"]])
|
|
454
|
+
out.append(new_tag)
|
|
455
|
+
last = el["end"]
|
|
456
|
+
out.append(s[last:])
|
|
457
|
+
return "".join(out), count
|
|
358
458
|
|
|
359
459
|
|
|
360
460
|
@_outside_protected
|
|
@@ -242,6 +242,13 @@ class TestRunObserve(unittest.TestCase):
|
|
|
242
242
|
self.assertEqual(art["preview"]["books"][0]["path"], "a.epub")
|
|
243
243
|
self.assertEqual(art["codes"]["occurrences"]["RSC-005"], 1)
|
|
244
244
|
self.assertNotIn("trend", art)
|
|
245
|
+
# the per-book census rows: every book lands here, clean or not
|
|
246
|
+
rows = {b["path"]: b for b in art["books"]}
|
|
247
|
+
self.assertEqual(set(rows), {"a.epub", "b.epub"})
|
|
248
|
+
self.assertEqual(rows["a.epub"]["errors"], 2)
|
|
249
|
+
self.assertEqual(rows["a.epub"]["codes"], ["RSC-005"])
|
|
250
|
+
self.assertEqual(rows["b.epub"]["codes"], [])
|
|
251
|
+
self.assertFalse(rows["b.epub"]["unreadable"])
|
|
245
252
|
|
|
246
253
|
def test_no_preview_omits_the_block(self):
|
|
247
254
|
rc = self._run(no_preview=True)
|
|
@@ -288,6 +288,37 @@ class TestPrologAndXmlns(unittest.TestCase):
|
|
|
288
288
|
out, n = strip_prolog_junk("<!DOCTYPE html><html/>")
|
|
289
289
|
self.assertEqual((out, n), ("<!DOCTYPE html><html/>", 1))
|
|
290
290
|
|
|
291
|
+
def test_calibre_cdata_style_wrapper_is_never_amputated(self):
|
|
292
|
+
# the 2026-10-24 surgery: Calibre wraps inline CSS in
|
|
293
|
+
# /*<![CDATA[*/ ... /*]]>*/; the bare CDATA protection ended at the
|
|
294
|
+
# first ]]>, leaving the trailing */ as an outside segment that
|
|
295
|
+
# strip_prolog_junk amputated (a phantom fix truncating the CSS
|
|
296
|
+
# comment: one CSS-008 "Premature end of file" per wrapped file,
|
|
297
|
+
# 111 gate-rejected books library-wide)
|
|
298
|
+
s = (
|
|
299
|
+
'<?xml version="1.0"?><html><head><style type="text/css">'
|
|
300
|
+
"/*<![CDATA[*/ p { color: red } /*]]>*/</style></head>"
|
|
301
|
+
"<body/></html>"
|
|
302
|
+
)
|
|
303
|
+
self.assertEqual(strip_prolog_junk(s), (s, 0))
|
|
304
|
+
self.assertIn("/*]]>*/", s)
|
|
305
|
+
|
|
306
|
+
def test_truncated_calibre_wrapper_still_reachable_for_the_terminator_fix(self):
|
|
307
|
+
# a genuinely truncated closer (no trailing */) must stay outside the
|
|
308
|
+
# protected span so --fix-cdata-terminator can still see and repair it
|
|
309
|
+
s = (
|
|
310
|
+
'<?xml version="1.0"?><html><head><style type="text/css">'
|
|
311
|
+
"/*<![CDATA[*/ p { color: red } /*]]></style></head>"
|
|
312
|
+
"<body/></html>"
|
|
313
|
+
)
|
|
314
|
+
self.assertEqual(strip_prolog_junk(s), (s, 0))
|
|
315
|
+
self.assertNotIn("/*]]>*/", s) # the truncated shape, preserved as-is
|
|
316
|
+
|
|
317
|
+
def test_real_junk_still_stripped(self):
|
|
318
|
+
out, n = strip_prolog_junk("\xef\xbb\xbfstray bytes<html/>")
|
|
319
|
+
self.assertEqual(n, 1)
|
|
320
|
+
self.assertTrue(out.startswith("<html/>"))
|
|
321
|
+
|
|
291
322
|
def test_junk_stripped_without_a_declaration(self):
|
|
292
323
|
out, n = strip_prolog_junk("stray<html/>")
|
|
293
324
|
self.assertEqual((out, n), ("<html/>", 1))
|
|
@@ -530,12 +561,87 @@ class TestFixNcxPlayorder(unittest.TestCase):
|
|
|
530
561
|
self.assertEqual(out, ncx)
|
|
531
562
|
|
|
532
563
|
def test_resequence_count_still_counts_real_changes(self):
|
|
564
|
+
# contract semantics (2026-10-24): two playOrder-bearing elements with
|
|
565
|
+
# no <content> are distinct targets, so sharing one value violates the
|
|
566
|
+
# distinctness rule; resequencing changes only the first (1) and the
|
|
567
|
+
# second lands on 2 unchanged, so one attribute change
|
|
533
568
|
ncx = '<navPoint playOrder="2"/><navPoint playOrder="2"/>'
|
|
534
569
|
out, n = fix_ncx_playorder(ncx)
|
|
535
|
-
self.assertEqual(n, 1)
|
|
570
|
+
self.assertEqual(n, 1)
|
|
536
571
|
self.assertIn('playOrder="1"', out)
|
|
537
572
|
self.assertIn('playOrder="2"', out)
|
|
538
573
|
|
|
574
|
+
def test_shared_target_valid_ncx_is_never_rewritten(self):
|
|
575
|
+
# the steel-breeze shape from the 2026-10-24 surgery: two navPoints
|
|
576
|
+
# whose <content src> point at the same file legitimately share one
|
|
577
|
+
# playOrder; the old unconditional renumbering split them into a
|
|
578
|
+
# finding and the gate rejected the book
|
|
579
|
+
ncx = (
|
|
580
|
+
'<navPoint id="num_1" playOrder="1"><content src="Text/split_000.html"/></navPoint>'
|
|
581
|
+
'<navPoint id="num_2" playOrder="1"><content src="Text/split_000.html"/></navPoint>'
|
|
582
|
+
)
|
|
583
|
+
out, n = fix_ncx_playorder(ncx)
|
|
584
|
+
self.assertEqual((out, n), (ncx, 0))
|
|
585
|
+
|
|
586
|
+
def test_pagetarget_participates_in_the_sequence(self):
|
|
587
|
+
# chronomancy's shape: valid navPoints 1..3 with a pageTarget pointing
|
|
588
|
+
# at its own page far ahead is a sequence gap; the pageTarget joins
|
|
589
|
+
# the sequence as value 4
|
|
590
|
+
ncx = (
|
|
591
|
+
'<navPoint playOrder="1"><content src="a.xhtml"/></navPoint>'
|
|
592
|
+
'<navPoint playOrder="2"><content src="b.xhtml"/></navPoint>'
|
|
593
|
+
'<navPoint playOrder="3"><content src="c.xhtml"/></navPoint>'
|
|
594
|
+
'<pageTarget id="page_447" value="447" playOrder="442">'
|
|
595
|
+
'<content src="page447.xhtml"/></pageTarget>'
|
|
596
|
+
)
|
|
597
|
+
out, n = fix_ncx_playorder(ncx)
|
|
598
|
+
self.assertEqual(n, 1)
|
|
599
|
+
self.assertIn('playOrder="4"', out)
|
|
600
|
+
self.assertNotIn('playOrder="442"', out)
|
|
601
|
+
|
|
602
|
+
def test_pagetarget_sequence_valid_is_untouched(self):
|
|
603
|
+
ncx = (
|
|
604
|
+
'<navPoint playOrder="1"><content src="a.xhtml"/></navPoint>'
|
|
605
|
+
'<navPoint playOrder="2"><content src="b.xhtml"/></navPoint>'
|
|
606
|
+
'<pageTarget id="pg" value="3" playOrder="3">'
|
|
607
|
+
'<content src="c.xhtml"/></pageTarget>'
|
|
608
|
+
)
|
|
609
|
+
out, n = fix_ncx_playorder(ncx)
|
|
610
|
+
self.assertEqual((out, n), (ncx, 0))
|
|
611
|
+
|
|
612
|
+
def test_same_target_conflicting_values_share_one_value(self):
|
|
613
|
+
ncx = (
|
|
614
|
+
'<navPoint playOrder="1"><content src="a.xhtml"/></navPoint>'
|
|
615
|
+
'<navPoint playOrder="2"><content src="a.xhtml"/></navPoint>'
|
|
616
|
+
'<navPoint playOrder="3"><content src="b.xhtml"/></navPoint>'
|
|
617
|
+
)
|
|
618
|
+
out, n = fix_ncx_playorder(ncx)
|
|
619
|
+
# groups in first-appearance order: a.xhtml -> 1 (shared by the first
|
|
620
|
+
# two), b.xhtml -> 2; both the conflict (2) and the old 3 are rewritten
|
|
621
|
+
self.assertEqual(n, 2)
|
|
622
|
+
self.assertEqual(out.count('playOrder="1"'), 2)
|
|
623
|
+
self.assertIn('playOrder="2"', out)
|
|
624
|
+
self.assertNotIn('playOrder="3"', out)
|
|
625
|
+
|
|
626
|
+
def test_nested_navpoints_with_parent_content_first_are_untouched(self):
|
|
627
|
+
ncx = (
|
|
628
|
+
'<navPoint id="a" playOrder="1"><navLabel><text>A</text></navLabel>'
|
|
629
|
+
'<content src="a.xhtml"/>'
|
|
630
|
+
'<navPoint id="b" playOrder="2"><navLabel><text>B</text></navLabel>'
|
|
631
|
+
'<content src="b.xhtml"/></navPoint></navPoint>'
|
|
632
|
+
)
|
|
633
|
+
out, n = fix_ncx_playorder(ncx)
|
|
634
|
+
self.assertEqual((out, n), (ncx, 0))
|
|
635
|
+
|
|
636
|
+
def test_non_integer_playorder_is_resequenced(self):
|
|
637
|
+
ncx = (
|
|
638
|
+
'<navPoint playOrder="x"><content src="a.xhtml"/></navPoint>'
|
|
639
|
+
'<navPoint playOrder="2"><content src="b.xhtml"/></navPoint>'
|
|
640
|
+
)
|
|
641
|
+
out, n = fix_ncx_playorder(ncx)
|
|
642
|
+
self.assertEqual(n, 1)
|
|
643
|
+
self.assertIn('playOrder="1"', out)
|
|
644
|
+
|
|
539
645
|
|
|
540
646
|
class TestStripInvalidValue(unittest.TestCase):
|
|
541
647
|
def test_misplaced_value_is_stripped(self):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|