bindery-cli 0.33.0__tar.gz → 0.35.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/PKG-INFO +3 -3
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/README.md +2 -2
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/patchnotes.md +72 -3
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/pyproject.toml +1 -1
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/roadmap.md +38 -2
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/spec.md +38 -11
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/__init__.py +1 -1
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/cli.py +6 -1
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/validate.py +48 -16
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_cli.py +43 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_validate.py +100 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/uv.lock +1 -1
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/.github/workflows/ci.yml +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/.github/workflows/publish.yml +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/.gitignore +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/AGENTS.md +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/CLAUDE.md +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/LICENSE +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/docs/screenshots/library-sweep.png +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/logo.svg +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/run_tests.sh +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/scripts/FastSweep.java +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/scripts/fast_sweep.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/__main__.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/audit.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/epub.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/library.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/pagination.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/reserialize.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/transforms.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/src/bindery/watermark.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/testing_facility/books_to_fix/.gitkeep +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/testing_facility/top500candidates/REPORT.md +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_audit.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_epub.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_library.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_pagination.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_reserialize.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_transforms.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_version.py +0 -0
- {bindery_cli-0.33.0 → bindery_cli-0.35.0}/tests/test_watermark.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bindery-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.35.0
|
|
4
4
|
Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
|
|
5
5
|
Author: vrnvctss
|
|
6
6
|
License: MIT
|
|
@@ -82,7 +82,7 @@ Every repair is gated by [epubcheck](https://github.com/w3c/epubcheck). The acce
|
|
|
82
82
|
|
|
83
83
|
Introducing a net-new fatal is always rejected. If epubcheck itself fails to run (crash, timeout, unparsable output), the book is reported as an error and never applied; only an explicit `--no-validate` skips the gate. Originals are never modified except by an explicit, atomic in-place replace (see below), and even then only after the gate accepts the result.
|
|
84
84
|
|
|
85
|
-
Under the hood, the oracle can run two ways: the usual `epubcheck --json -` subprocess, or a
|
|
85
|
+
Under the hood, the oracle can run two ways: the usual `epubcheck --json -` subprocess, or a persistent daemon that writes `FastDaemon.java` into a temp directory at runtime, compiles it in memory with the running JVM's own compiler (Java's single-file source launcher: no javac, no javac/java version skew), and drives the warm JVM over a pipe. The daemon reports the counts it reads from the JSON document that epubcheck's own `CheckingReport` serializes, so it measures exactly what the subprocess measures, just ~15x faster. It is a pool bounded by `--workers` (serial runs use one), every roundtrip is bounded by the caller's timeout, and any failure tears it down and falls back to the subprocess for good.
|
|
86
86
|
|
|
87
87
|
The lossy modes (`--strip-pagination`, `--strip-broken-tags`, and `--strip-watermarks`) are the exception to the "must improve" rule. Since they remove visible markup rather than correcting XML schema violations, epubcheck counts often remain unchanged. They are accepted when the result is **no worse** (no net-new fatals or errors), relying on strict programmatic safety nets instead.
|
|
88
88
|
|
|
@@ -209,7 +209,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
|
|
|
209
209
|
- Only the `.epub` is replaced. `metadata.opf`, `cover.jpg`, and `metadata.db` are left for Calibre's Quality Check sync to reconcile.
|
|
210
210
|
- A per-book progress line goes to stderr (stdout stays a clean report); `--quiet` suppresses it. A corrupt or unreadable book is reported and skipped, never aborting the sweep.
|
|
211
211
|
- Exit codes: 0 for a clean sweep, 1 for a usage error, 2 when any book was rejected, unreadable, or failed epubcheck (for scripts and cron). Note that argparse-level misuse (an unknown flag or a malformed argument) exits with 2 before any validation runs; the tool's own usage validations exit 1.
|
|
212
|
-
- A `partial` book (improved but still unable to open) is reported for manual follow-up and
|
|
212
|
+
- A `partial` book (improved but still unable to open) is reported for manual follow-up and fails the run with exit 2, in `library`, `run phase1`, and `run phase3` alike: a book that still needs a human is trouble.
|
|
213
213
|
- `repair` refuses to overwrite an existing output file unless `--force` is given.
|
|
214
214
|
|
|
215
215
|
## Run slices
|
|
@@ -60,7 +60,7 @@ Every repair is gated by [epubcheck](https://github.com/w3c/epubcheck). The acce
|
|
|
60
60
|
|
|
61
61
|
Introducing a net-new fatal is always rejected. If epubcheck itself fails to run (crash, timeout, unparsable output), the book is reported as an error and never applied; only an explicit `--no-validate` skips the gate. Originals are never modified except by an explicit, atomic in-place replace (see below), and even then only after the gate accepts the result.
|
|
62
62
|
|
|
63
|
-
Under the hood, the oracle can run two ways: the usual `epubcheck --json -` subprocess, or a
|
|
63
|
+
Under the hood, the oracle can run two ways: the usual `epubcheck --json -` subprocess, or a persistent daemon that writes `FastDaemon.java` into a temp directory at runtime, compiles it in memory with the running JVM's own compiler (Java's single-file source launcher: no javac, no javac/java version skew), and drives the warm JVM over a pipe. The daemon reports the counts it reads from the JSON document that epubcheck's own `CheckingReport` serializes, so it measures exactly what the subprocess measures, just ~15x faster. It is a pool bounded by `--workers` (serial runs use one), every roundtrip is bounded by the caller's timeout, and any failure tears it down and falls back to the subprocess for good.
|
|
64
64
|
|
|
65
65
|
The lossy modes (`--strip-pagination`, `--strip-broken-tags`, and `--strip-watermarks`) are the exception to the "must improve" rule. Since they remove visible markup rather than correcting XML schema violations, epubcheck counts often remain unchanged. They are accepted when the result is **no worse** (no net-new fatals or errors), relying on strict programmatic safety nets instead.
|
|
66
66
|
|
|
@@ -187,7 +187,7 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
|
|
|
187
187
|
- Only the `.epub` is replaced. `metadata.opf`, `cover.jpg`, and `metadata.db` are left for Calibre's Quality Check sync to reconcile.
|
|
188
188
|
- A per-book progress line goes to stderr (stdout stays a clean report); `--quiet` suppresses it. A corrupt or unreadable book is reported and skipped, never aborting the sweep.
|
|
189
189
|
- Exit codes: 0 for a clean sweep, 1 for a usage error, 2 when any book was rejected, unreadable, or failed epubcheck (for scripts and cron). Note that argparse-level misuse (an unknown flag or a malformed argument) exits with 2 before any validation runs; the tool's own usage validations exit 1.
|
|
190
|
-
- A `partial` book (improved but still unable to open) is reported for manual follow-up and
|
|
190
|
+
- A `partial` book (improved but still unable to open) is reported for manual follow-up and fails the run with exit 2, in `library`, `run phase1`, and `run phase3` alike: a book that still needs a human is trouble.
|
|
191
191
|
- `repair` refuses to overwrite an existing output file unless `--force` is given.
|
|
192
192
|
|
|
193
193
|
## Run slices
|
|
@@ -1,4 +1,73 @@
|
|
|
1
1
|
# bindery-cli Patch Notes
|
|
2
|
+
## v0.35.0 (2026-09-10)
|
|
3
|
+
|
|
4
|
+
### The epubcheck daemon, reconciled: oracle-identical counts at ~15x the speed
|
|
5
|
+
|
|
6
|
+
- **FastDaemon v2 measures exactly what the subprocess oracle measures.**
|
|
7
|
+
The daemon's old Java side counted message occurrences from
|
|
8
|
+
`CheckingReport`'s counters, while the gate has always measured the
|
|
9
|
+
counts in epubcheck's own `--json` output, which aggregates identical
|
|
10
|
+
messages (one book: 243 occurrences, 4 aggregated messages). On the
|
|
11
|
+
staged-fixture corpus the two methods diverged on 11 of 14 books, so
|
|
12
|
+
the daemon could never be enabled without changing what the gate
|
|
13
|
+
measures. The new daemon calls `CheckingReport.generate()` and reads
|
|
14
|
+
`nFatal`/`nError`/`nWarning` out of the JSON epubcheck itself
|
|
15
|
+
serializes: identical numbers by construction. Verified against the
|
|
16
|
+
subprocess oracle on all 14 staged books and on a 100-book
|
|
17
|
+
real-library sample: 114/114 identical.
|
|
18
|
+
- **No more javac, no more version skew.** The daemon now launches
|
|
19
|
+
through Java's single-file source launcher (JEP 330): the running JVM
|
|
20
|
+
compiles the daemon in memory with its own compiler, so the
|
|
21
|
+
javac-newer-than-java skew that kept the daemon dead on this machine
|
|
22
|
+
(javac 27-ea, java 25) cannot recur on any toolchain.
|
|
23
|
+
- **Measured, working daemon throughput: ~0.27s/book warm against
|
|
24
|
+
~4.2s/book for the subprocess oracle** (15.6x). With `--workers N`
|
|
25
|
+
each worker gets its own warm daemon from the bounded pool, and every
|
|
26
|
+
roundtrip stays bounded by the caller's timeout with the fail-safe
|
|
27
|
+
fallback to the subprocess oracle.
|
|
28
|
+
- **The dedup semantics are pinned by a suite test:** a book engineered
|
|
29
|
+
with one defect repeated three times plus one different defect must
|
|
30
|
+
answer 2 errors (the aggregated count), not 4 occurrences (the old
|
|
31
|
+
daemon's answer). Skipped where no epubcheck/JDK exists, like the
|
|
32
|
+
other oracle tests. README and spec's gate sections describe the new
|
|
33
|
+
design.
|
|
34
|
+
|
|
35
|
+
## v0.34.0 (2026-09-10)
|
|
36
|
+
|
|
37
|
+
### Brandon's three calls: partial books are trouble, the contract reconciled, the daemon decision informed
|
|
38
|
+
|
|
39
|
+
- **Partial books now fail every layer (exit-code decision, option A).**
|
|
40
|
+
A `partial` book (repaired, improved, but still unable to open) used to
|
|
41
|
+
exit 0 from `library` and `run phase3` while `run phase1` counted it
|
|
42
|
+
as trouble. All three layers now exit 2 on it: the run succeeded, but
|
|
43
|
+
a book still needs a human. README and spec updated to match, and a
|
|
44
|
+
regression test pins a 3f-to-1f book exiting 2 with `applied: 0`.
|
|
45
|
+
- **spec.md reconciled with the shipped opt-ins** (applied on Brandon's
|
|
46
|
+
go, from the draft filed in v0.33.0's roadmap box). The non-goals no
|
|
47
|
+
longer prohibit what ships: "fixing RSC-005 in bulk" now names the
|
|
48
|
+
scoped, version-gated repairs as the shipped exception, and "repairing
|
|
49
|
+
genuinely mangled structure" now describes `--reserialize`'s actual
|
|
50
|
+
bounds (html root required, non-HTML XML sidecars refused). The
|
|
51
|
+
missing `--fix-ids` OPF section is written (manifest item ids, spine
|
|
52
|
+
idref/toc, fallback, media-overlay, EPUB 2 cover meta, both quote
|
|
53
|
+
styles), the garbled NCX sentence is rewritten, and "the OPF is left
|
|
54
|
+
untouched" now reads "by the default pass", naming the five opt-ins
|
|
55
|
+
that edit it on request.
|
|
56
|
+
- **The daemon decision, informed:** Brandon asked whether upgrading
|
|
57
|
+
java to 27 fixes the epubcheck daemon. Investigated and answered: his
|
|
58
|
+
javac is Fedora's `java-latest-openjdk-devel` 27-ea while `java` is
|
|
59
|
+
`java-25-openjdk` (both official Fedora packages, no COPR involved),
|
|
60
|
+
but the version skew is not the real blocker. epubcheck's own JSON
|
|
61
|
+
output is deduplicated (it reports 4 errors for a book whose CLI
|
|
62
|
+
summary says 243), the daemon counts everything, and bindery's gate
|
|
63
|
+
has always measured with JSON-style counts, so a working daemon would
|
|
64
|
+
change what the gate measures. The skew itself is fixable in code
|
|
65
|
+
(Java's single-file source launcher needs no javac at all), but the
|
|
66
|
+
daemon should stay off until its counting matches the JSON oracle; it
|
|
67
|
+
already self-disables safely, and `--workers` parallelism works via
|
|
68
|
+
subprocesses. Retire-or-reconcile remains the open decision; recorded
|
|
69
|
+
in the roadmap's `:958` box.
|
|
70
|
+
|
|
2
71
|
## v0.33.0 (2026-09-10)
|
|
3
72
|
|
|
4
73
|
### Phase 14 hardening: the apply path, the audit, and the safety contract's tests
|
|
@@ -24,14 +93,14 @@ regression test.
|
|
|
24
93
|
bounded pool sized by `--workers` (default 1: the old behavior), a
|
|
25
94
|
busy pool falls back to subprocesses instead of blocking, every
|
|
26
95
|
roundtrip is bounded by the caller's timeout, and a daemon that dies
|
|
27
|
-
or wedges is torn down and never retried
|
|
28
|
-
the pool so the sweep degrades to pure subprocess checks instead of
|
|
96
|
+
or wedges is torn down and never retried; one failed attempt exhausts
|
|
97
|
+
the pool, so the sweep degrades to pure subprocess checks instead of
|
|
29
98
|
respawning a JVM per book. Measured: 8 books, 4 workers, two ~10s
|
|
30
99
|
waves. The evidence for the recorded daemon-classpath decision grew:
|
|
31
100
|
this machine's javac (27-ea) and java (25) cannot load each other's
|
|
32
101
|
classes, so the daemon has never served a count here, and when forced
|
|
33
102
|
to run its counts diverge from the CLI JSON oracle (243 vs 4 on one
|
|
34
|
-
book)
|
|
103
|
+
book): enabling it would change gate outcomes.
|
|
35
104
|
- **Smaller apply papercuts:** `--limit < 1` and a missing `--audit`
|
|
36
105
|
file are usage errors, not tracebacks; the fresh-format install
|
|
37
106
|
registers the catalogued file's own name instead of `repaired`; and
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "bindery-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.35.0"
|
|
8
8
|
description = "Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
classifiers = [
|
|
@@ -1068,7 +1068,7 @@ can be net-neutral and ships silently.*
|
|
|
1068
1068
|
and JSON with exit 2; the resumable run journal is deliberately not
|
|
1069
1069
|
built, since isolation plus `--json` already preserves the record
|
|
1070
1070
|
and the journal is a second write surface next to the library.)*
|
|
1071
|
-
- [
|
|
1071
|
+
- [x] **Drive the exit code from partial books.** The documented contract
|
|
1072
1072
|
says exit 2 means "ran fine but some books are in trouble", but
|
|
1073
1073
|
partial books land in `still_fatal` and return 0, in both `library`
|
|
1074
1074
|
and phase3, while phase1 maps partial to problem/exit 2; the layers
|
|
@@ -1089,6 +1089,11 @@ can be net-neutral and ships silently.*
|
|
|
1089
1089
|
cost: scripts that care must parse the report, and the contract's
|
|
1090
1090
|
exit-2 sentence stays subtly incomplete.
|
|
1091
1091
|
Either way the exit-code documentation lands with the decision.
|
|
1092
|
+
*(DECIDED 2026-09-10: option A, "unify on trouble". `run_library`
|
|
1093
|
+
now folds `partials` into its trouble exit, and phase3 propagates
|
|
1094
|
+
that rc, so all three layers exit 2 on a partial book. README and
|
|
1095
|
+
spec updated to match; regression test pins a 3f->1f book exiting
|
|
1096
|
+
2 with applied=0. Shipped in v0.34.0.)*
|
|
1092
1097
|
- [x] **Give backups overwrite protection and keep them out of the
|
|
1093
1098
|
candidate set.** `make_backup` clobbers an existing backup, so a
|
|
1094
1099
|
second `--apply` destroys the only copy of the author original
|
|
@@ -1137,7 +1142,38 @@ can be net-neutral and ships silently.*
|
|
|
1137
1142
|
`--only ncx` encrypted-archive crash is also fixed here:
|
|
1138
1143
|
`ncx_uid_mismatch` now catches the RuntimeError that `z.read`
|
|
1139
1144
|
raises on zip-encrypted entries, so a DRM'd book is not a candidate
|
|
1140
|
-
rather than a crash.)
|
|
1145
|
+
rather than a crash.)
|
|
1146
|
+
*(DECISION NOTE 2026-09-10: Brandon asked whether upgrading his
|
|
1147
|
+
java to 27 would fix the daemon. Answered no: his javac is Fedora's
|
|
1148
|
+
java-latest-openjdk-devel 27-ea while java is java-25-openjdk (both
|
|
1149
|
+
official Fedora packages, no COPR involved), but the version skew
|
|
1150
|
+
is not the real blocker: epubcheck's own JSON output is
|
|
1151
|
+
deduplicated (it reports 4 errors for a book its CLI summary calls
|
|
1152
|
+
243), the daemon counts everything, and bindery's gate has always
|
|
1153
|
+
measured with JSON-style counts. Upgrading java would make the
|
|
1154
|
+
daemon boot and start feeding different numbers into the gate. The
|
|
1155
|
+
skew itself is fixable in code (single-file source launcher, no
|
|
1156
|
+
javac at all), but the daemon should stay off until its counting
|
|
1157
|
+
matches the JSON oracle; retire-or-reconcile remains the open
|
|
1158
|
+
decision. The daemon already self-disables safely, and --workers
|
|
1159
|
+
parallelism works via subprocesses.)
|
|
1160
|
+
*(DECIDED 2026-09-10: Brandon chose RECONCILE over retire. The
|
|
1161
|
+
counting divergence turned out to run deeper than the daemon: the
|
|
1162
|
+
CLI's human summary counts message occurrences (243 on one book)
|
|
1163
|
+
while epubcheck's own --json output counts aggregated messages
|
|
1164
|
+
(4 on the same book), and the gate has always measured the
|
|
1165
|
+
aggregated scale. The reconciliation is therefore by construction:
|
|
1166
|
+
FastDaemon v2 calls CheckingReport.generate() and reads nFatal/
|
|
1167
|
+
nError/nWarning from the JSON epubcheck itself serializes, so the
|
|
1168
|
+
daemon measures the subprocess oracle's numbers exactly. Verified
|
|
1169
|
+
against the subprocess oracle on all 14 staged books (the same
|
|
1170
|
+
corpus that diverged 11/14 under the old daemon) and on a
|
|
1171
|
+
100-book real-library sample. The launch switched to Java's
|
|
1172
|
+
single-file source launcher, removing javac and the version skew
|
|
1173
|
+
permanently; the dedup semantics are pinned by a suite test
|
|
1174
|
+
(skipped where no epubcheck/JDK exists). Daemon warm throughput
|
|
1175
|
+
measured at ~0.27s/book vs ~4.2s/book subprocess. Shipped in
|
|
1176
|
+
v0.35.0.)**
|
|
1141
1177
|
- [x] **Smaller apply papercuts:** `--limit -1` and a missing
|
|
1142
1178
|
`--audit` file raise tracebacks instead of usage errors
|
|
1143
1179
|
(`cli.py:434`, `373`); the fresh-format branch of `install_format`
|
|
@@ -50,7 +50,11 @@ Applied to content documents (`.xhtml`, `.html`, `.htm`, `.xml`), in order:
|
|
|
50
50
|
change what renders. Removed end tags are counted in the fix total.
|
|
51
51
|
|
|
52
52
|
Applied to the NCX sidecar (`.ncx`): strip_prolog_junk, escape_bare_amp,
|
|
53
|
-
fix_named_entities, plus **dtb:uid sync
|
|
53
|
+
fix_named_entities, plus **dtb:uid sync** (rewriting the NCX's `dtb:uid`
|
|
54
|
+
meta to match the OPF's unique identifier when they drift, NCX-001) and
|
|
55
|
+
**fix_ncx_playorder** (resequencing `playOrder` attributes on
|
|
56
|
+
`<navPoint>` start tags so they are strictly sequential; nav-label text
|
|
57
|
+
is never touched).
|
|
54
58
|
With `--fix-ids`, ids in the NCX that are not valid XML names (digit-led, as when a
|
|
55
59
|
converter stamps navPoint ids from UUIDs; colon-bearing) are renamed with the same
|
|
56
60
|
`id_` scheme as OPF manifest ids. NCX ids are internal to the NCX (nothing in the
|
|
@@ -58,7 +62,10 @@ OPF or content documents references them), so the rename needs no cross-file
|
|
|
58
62
|
bookkeeping.
|
|
59
63
|
|
|
60
64
|
The OPF is located via `META-INF/container.xml` (falling back to the first `.opf`
|
|
61
|
-
in the archive)
|
|
65
|
+
in the archive). The default pass leaves it untouched, to keep Calibre's embedded
|
|
66
|
+
metadata pristine; the opt-in `--fix-ids`, `--strip-epub3-attrs`, `--fix-page-map`,
|
|
67
|
+
`--prune-missing-resources`, and `--encode-url-spaces` each edit it when requested,
|
|
68
|
+
and the human-facing `dc:` metadata is never altered by any of them.
|
|
62
69
|
|
|
63
70
|
### Opt-in: escape unknown entities (`--escape-unknown-entities`)
|
|
64
71
|
|
|
@@ -97,6 +104,12 @@ part of the default pipeline:
|
|
|
97
104
|
external URL names a position in that other document and survives verbatim, and the
|
|
98
105
|
NCX's `content src` fragments follow the rename (`fix_ncx_src_fragments`), so a ToC
|
|
99
106
|
never dangles against the ids it references.
|
|
107
|
+
- **`--fix-ids`**: renames manifest `item` ids that are not valid XML names
|
|
108
|
+
(digit-led, colon-bearing) with the deterministic `id_` scheme, and updates
|
|
109
|
+
every reference to them: spine `idref` and `toc`, item `fallback` and
|
|
110
|
+
`media-overlay`, and the EPUB 2 cover meta, in both quote styles. NCX ids are
|
|
111
|
+
renamed by the same pass (`fix_ncx_ids`); colon-bearing fragments follow the
|
|
112
|
+
rename via `--fix-id-colons`' NCX half. Href paths and filenames are untouched.
|
|
100
113
|
- **`--unwrap-block-in-inline`**: drop a `<span>` that illegally wraps a
|
|
101
114
|
`<div>/<p>/<blockquote>`, keeping the block element and its text.
|
|
102
115
|
- **`--strip-invalid-value`**: remove misplaced `value="..."` attributes from non-form
|
|
@@ -226,10 +239,16 @@ the gate has not accepted anything, so nothing is applied or written. With
|
|
|
226
239
|
`--no-validate`, the gate is skipped and repairs are trusted on the RepairReport alone.
|
|
227
240
|
|
|
228
241
|
The oracle itself runs two ways: the `epubcheck --json -` subprocess, or a persistent
|
|
229
|
-
daemon (`_EpubcheckDaemon`, a bounded pool sized by `--workers`) that
|
|
230
|
-
`FastDaemon.java` at runtime and
|
|
231
|
-
|
|
232
|
-
|
|
242
|
+
daemon (`_EpubcheckDaemon`, a bounded pool sized by `--workers`) that writes
|
|
243
|
+
`FastDaemon.java` at runtime and launches it through Java's single-file source
|
|
244
|
+
launcher (in-memory compilation by the running JVM: no javac, no version skew),
|
|
245
|
+
driving the warm JVM over a pipe. The daemon answers with the counts it reads from
|
|
246
|
+
the JSON that epubcheck's own `CheckingReport.generate()` serializes — the same
|
|
247
|
+
document `--json` produces — so both paths measure identically by construction.
|
|
248
|
+
(Epubcheck's JSON counts aggregated messages, not occurrences: a defect repeated
|
|
249
|
+
three times is one error. The gate has always measured that scale.) Every daemon
|
|
250
|
+
roundtrip is bounded by the caller's timeout, and any daemon failure tears it down
|
|
251
|
+
and falls back to the subprocess for good.
|
|
233
252
|
|
|
234
253
|
## Page-number strip (opt-in, lossy)
|
|
235
254
|
|
|
@@ -302,8 +321,8 @@ For a Calibre library (`Author/Title (id)/Title - Author.epub`):
|
|
|
302
321
|
book was rejected, unreadable, or failed epubcheck, so scripts can detect trouble.
|
|
303
322
|
Argparse-level misuse (unknown flag, malformed argument) exits 2 before the tool's
|
|
304
323
|
own validation runs; the tool's own usage validations exit 1. A `partial` book
|
|
305
|
-
(improved but still fatal) is reported for manual follow-up and
|
|
306
|
-
|
|
324
|
+
(improved but still fatal) is reported for manual follow-up and counts as trouble:
|
|
325
|
+
`library`, `run phase1`, and `run phase3` all exit 2 on it (unified 2026-09-10).
|
|
307
326
|
- With `--sweep`, `--workers N` runs the candidate-selection epubcheck pass through N
|
|
308
327
|
concurrent workers (default 1: serial, unchanged). Books are checked in windows of N
|
|
309
328
|
consumed in input order, so the candidate set and the before-measurements are identical
|
|
@@ -376,7 +395,15 @@ verb call). Books left partial or unreadable surface as `decisions_needed`.
|
|
|
376
395
|
|
|
377
396
|
## Out of scope (non-goals)
|
|
378
397
|
|
|
379
|
-
- Fixing RSC-005 schema/content-model violations in bulk
|
|
380
|
-
-
|
|
381
|
-
|
|
398
|
+
- Fixing RSC-005 schema/content-model violations *in bulk*: the scoped, opt-in
|
|
399
|
+
RSC-005 repairs (`--strip-epub3-attrs`, `--downgrade-epub3-tags`, both gated on
|
|
400
|
+
the package version) ship and stay, but an indiscriminate RSC-005 sweep remains
|
|
401
|
+
out of scope.
|
|
402
|
+
- Repairing genuinely mangled structure remains opt-in and bounded:
|
|
403
|
+
`--reserialize` (the lone html5lib dependency) re-parses malformed documents
|
|
404
|
+
carrying an `<html>` root and re-emits XHTML, refusing non-HTML XML sidecars;
|
|
405
|
+
corrupted tag names are covered by `--strip-bad-attrs` and
|
|
406
|
+
`--unwrap-illegal-tags`. Wholesale structural rewrites without a root to anchor
|
|
407
|
+
on stay out of scope, and anything the opt-ins cannot make deterministically
|
|
408
|
+
safe is detected as `partial`/`nochange` and reported.
|
|
382
409
|
- Editing human-facing dc: metadata or creating content.
|
|
@@ -680,7 +680,12 @@ def run_library(args) -> int:
|
|
|
680
680
|
|
|
681
681
|
# 2 lets scripts and cron distinguish "ran fine but some books are in trouble"
|
|
682
682
|
# from a clean sweep (0) and a usage error (1).
|
|
683
|
-
|
|
683
|
+
# Exit codes per the documented contract, unified on "trouble" (Brandon's
|
|
684
|
+
# option-A call, 2026-09-10): a partial book (improved but still unable
|
|
685
|
+
# to open) IS trouble -- the run itself succeeded, but a book still needs
|
|
686
|
+
# a human. phase1 already mapped partial to 2; library and phase3 (which
|
|
687
|
+
# propagates this rc) now agree with it.
|
|
688
|
+
return 2 if (rejected + errors + unreadable + partials) > 0 else 0
|
|
684
689
|
|
|
685
690
|
|
|
686
691
|
def run_repair(args) -> int:
|
|
@@ -78,12 +78,39 @@ def _counts_from_json(stdout: str) -> CheckResult | None:
|
|
|
78
78
|
|
|
79
79
|
_DAEMON_JAVA = """
|
|
80
80
|
import java.io.File;
|
|
81
|
+
import java.io.StringWriter;
|
|
81
82
|
import java.io.PrintWriter;
|
|
82
83
|
import java.util.Scanner;
|
|
83
84
|
import com.adobe.epubcheck.api.EpubCheck;
|
|
84
85
|
import com.adobe.epubcheck.reporting.CheckingReport;
|
|
85
86
|
|
|
86
87
|
public class FastDaemon {
|
|
88
|
+
// Answers with the counts from the checker block of the JSON document
|
|
89
|
+
// that CheckingReport.generate() itself serializes: the exact JSON the
|
|
90
|
+
// epubcheck CLI's --json mode carries. Counting must match the
|
|
91
|
+
// subprocess oracle by construction, never by imitation: the human
|
|
92
|
+
// summary counts message occurrences while the JSON checker block
|
|
93
|
+
// counts aggregated messages, and the gate is calibrated on the latter.
|
|
94
|
+
static int field(String json, String name) {
|
|
95
|
+
int i = json.indexOf(name);
|
|
96
|
+
while (i != -1) {
|
|
97
|
+
int j = i + name.length();
|
|
98
|
+
while (j < json.length() && (json.charAt(j) == ' '
|
|
99
|
+
|| json.charAt(j) == ':' || json.charAt(j) == '"')) {
|
|
100
|
+
j++;
|
|
101
|
+
}
|
|
102
|
+
if (j < json.length() && Character.isDigit(json.charAt(j))) {
|
|
103
|
+
int k = j;
|
|
104
|
+
while (k < json.length() && Character.isDigit(json.charAt(k))) {
|
|
105
|
+
k++;
|
|
106
|
+
}
|
|
107
|
+
return Integer.parseInt(json.substring(j, k));
|
|
108
|
+
}
|
|
109
|
+
i = json.indexOf(name, i + 1);
|
|
110
|
+
}
|
|
111
|
+
return -1;
|
|
112
|
+
}
|
|
113
|
+
|
|
87
114
|
public static void main(String[] args) throws Exception {
|
|
88
115
|
Scanner scanner = new Scanner(System.in);
|
|
89
116
|
while (scanner.hasNextLine()) {
|
|
@@ -95,13 +122,20 @@ public class FastDaemon {
|
|
|
95
122
|
continue;
|
|
96
123
|
}
|
|
97
124
|
try {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
125
|
+
StringWriter json = new StringWriter();
|
|
126
|
+
CheckingReport report = new CheckingReport(new PrintWriter(json),
|
|
127
|
+
epub.getName());
|
|
128
|
+
report.initialize();
|
|
129
|
+
new EpubCheck(epub, report).doValidate();
|
|
130
|
+
report.generate();
|
|
131
|
+
int f = field(json.toString(), "nFatal");
|
|
132
|
+
int e = field(json.toString(), "nError");
|
|
133
|
+
int w = field(json.toString(), "nWarning");
|
|
134
|
+
if (f < 0 || e < 0 || w < 0) {
|
|
135
|
+
System.out.println("-1,-1,-1");
|
|
136
|
+
} else {
|
|
137
|
+
System.out.println(f + "," + e + "," + w);
|
|
138
|
+
}
|
|
105
139
|
} catch (Exception e) {
|
|
106
140
|
System.out.println("-1,-1,-1");
|
|
107
141
|
}
|
|
@@ -150,14 +184,12 @@ class _EpubcheckDaemon:
|
|
|
150
184
|
with open(java_file, "w") as f:
|
|
151
185
|
f.write(_DAEMON_JAVA)
|
|
152
186
|
|
|
153
|
-
subprocess.run(
|
|
154
|
-
["javac", "-cp", jar_path, java_file],
|
|
155
|
-
check=True,
|
|
156
|
-
capture_output=True,
|
|
157
|
-
)
|
|
158
|
-
|
|
159
187
|
self._proc = subprocess.Popen(
|
|
160
|
-
|
|
188
|
+
# JEP 330 single-file source launcher: the JVM compiles the
|
|
189
|
+
# daemon in memory with its own compiler, so there is no
|
|
190
|
+
# javac step and no javac/java version skew (which kept the
|
|
191
|
+
# daemon dead on machines where javac is newer than java).
|
|
192
|
+
["java", "-cp", f".:{jar_path}", java_file],
|
|
161
193
|
cwd=self.workdir,
|
|
162
194
|
stdin=subprocess.PIPE,
|
|
163
195
|
stdout=subprocess.PIPE,
|
|
@@ -166,8 +198,8 @@ class _EpubcheckDaemon:
|
|
|
166
198
|
atexit.register(self.stop)
|
|
167
199
|
return True
|
|
168
200
|
except Exception:
|
|
169
|
-
# Leave nothing behind: a failed
|
|
170
|
-
#
|
|
201
|
+
# Leave nothing behind: a failed start must not leak the
|
|
202
|
+
# tempdir, and the failure is final for this instance.
|
|
171
203
|
self.stop()
|
|
172
204
|
return False
|
|
173
205
|
|
|
@@ -1309,3 +1309,46 @@ class TestSweepWorkers(unittest.TestCase):
|
|
|
1309
1309
|
self.assertEqual(rc, 0)
|
|
1310
1310
|
# 1 sweep call + 1 after-validation on the single candidate
|
|
1311
1311
|
self.assertEqual(len(checked), 2)
|
|
1312
|
+
|
|
1313
|
+
|
|
1314
|
+
class TestPartialBooksAreTrouble(unittest.TestCase):
|
|
1315
|
+
"""Brandon's option-A call (2026-09-10): a partial book (improved but
|
|
1316
|
+
still unable to open) is trouble -- library and phase3 exit 2 on it,
|
|
1317
|
+
agreeing with phase1's existing mapping."""
|
|
1318
|
+
|
|
1319
|
+
def test_library_exits_2_on_a_partial_book(self):
|
|
1320
|
+
with tempfile.TemporaryDirectory() as td:
|
|
1321
|
+
root = Path(td)
|
|
1322
|
+
_phase1_book(root / "a.epub", broken=True)
|
|
1323
|
+
jout = root / "lib.json"
|
|
1324
|
+
results = [
|
|
1325
|
+
CheckResult(3, 0, 0), # the sweep's before-measurement
|
|
1326
|
+
CheckResult(1, 0, 0), # after: fewer fatals, still fatal
|
|
1327
|
+
]
|
|
1328
|
+
out, err = io.StringIO(), io.StringIO()
|
|
1329
|
+
with (
|
|
1330
|
+
mock.patch("bindery.cli.epubcheck_available", return_value=True),
|
|
1331
|
+
mock.patch("bindery.cli.run_epubcheck", side_effect=results),
|
|
1332
|
+
redirect_stdout(out),
|
|
1333
|
+
redirect_stderr(err),
|
|
1334
|
+
):
|
|
1335
|
+
rc = cli.run_library(
|
|
1336
|
+
build_parser().parse_args(
|
|
1337
|
+
[
|
|
1338
|
+
"library",
|
|
1339
|
+
str(root),
|
|
1340
|
+
"--sweep",
|
|
1341
|
+
"--only",
|
|
1342
|
+
"all",
|
|
1343
|
+
"--all",
|
|
1344
|
+
"--apply",
|
|
1345
|
+
"--json",
|
|
1346
|
+
str(jout),
|
|
1347
|
+
]
|
|
1348
|
+
)
|
|
1349
|
+
)
|
|
1350
|
+
data = json.loads(jout.read_text())
|
|
1351
|
+
self.assertEqual(rc, 2)
|
|
1352
|
+
self.assertEqual(data["summary"]["partial"], 1)
|
|
1353
|
+
# a partial book is never auto-applied
|
|
1354
|
+
self.assertEqual(data["summary"]["applied"], 0)
|
|
@@ -15,6 +15,7 @@ from __future__ import annotations
|
|
|
15
15
|
import json
|
|
16
16
|
import os
|
|
17
17
|
import subprocess
|
|
18
|
+
import tempfile
|
|
18
19
|
import unittest
|
|
19
20
|
import zipfile
|
|
20
21
|
from pathlib import Path
|
|
@@ -22,7 +23,10 @@ from unittest import mock
|
|
|
22
23
|
|
|
23
24
|
from bindery.validate import (
|
|
24
25
|
CheckResult,
|
|
26
|
+
_counts_from_json,
|
|
25
27
|
_DaemonPool,
|
|
28
|
+
_english_locale_env,
|
|
29
|
+
_EpubcheckDaemon,
|
|
26
30
|
_readline_timeout,
|
|
27
31
|
gate,
|
|
28
32
|
no_worse,
|
|
@@ -205,6 +209,102 @@ class ReadlineTimeout(unittest.TestCase):
|
|
|
205
209
|
_readline_timeout(fh, 5)
|
|
206
210
|
|
|
207
211
|
|
|
212
|
+
@unittest.skipUnless(_daemon_toolchain_available(), "epubcheck not installed")
|
|
213
|
+
class DaemonMatchesJsonOracle(unittest.TestCase):
|
|
214
|
+
"""The counting-parity contract, pinned on real books: the daemon must
|
|
215
|
+
answer with the SAME numbers the subprocess --json oracle produces,
|
|
216
|
+
because the gate is calibrated on those numbers. FastDaemon v2 extracts
|
|
217
|
+
them from the JSON document CheckingReport.generate() itself serializes,
|
|
218
|
+
so the match holds by construction; this test holds it empirically,
|
|
219
|
+
including on a book engineered with repeated identical defects (the
|
|
220
|
+
exact shape where the old occurrence-counting daemon diverged)."""
|
|
221
|
+
|
|
222
|
+
def setUp(self) -> None:
|
|
223
|
+
self.tmp = tempfile.TemporaryDirectory()
|
|
224
|
+
# A clean book carrying exactly two distinct defects, one of them
|
|
225
|
+
# repeated three times: the JSON oracle deduplicates the repeats, so
|
|
226
|
+
# the true count is 2 errors from 4 occurrences. The old
|
|
227
|
+
# occurrence-counting daemon answered 4 here; by-construction
|
|
228
|
+
# counting must answer 2.
|
|
229
|
+
path = Path(self.tmp.name) / "dedup.epub"
|
|
230
|
+
with zipfile.ZipFile(path, "w") as z:
|
|
231
|
+
z.writestr("mimetype", "application/epub+zip")
|
|
232
|
+
z.writestr(
|
|
233
|
+
"META-INF/container.xml",
|
|
234
|
+
'<container version="1.0" '
|
|
235
|
+
'xmlns="urn:oasis:names:tc:opendocument:xmlns:container">'
|
|
236
|
+
'<rootfiles><rootfile full-path="content.opf" '
|
|
237
|
+
'media-type="application/oebps-package+xml"/></rootfiles>'
|
|
238
|
+
"</container>",
|
|
239
|
+
)
|
|
240
|
+
z.writestr(
|
|
241
|
+
"content.opf",
|
|
242
|
+
'<package xmlns="http://www.idpf.org/2007/opf" version="2.0" '
|
|
243
|
+
'unique-identifier="bid"><metadata '
|
|
244
|
+
'xmlns:dc="http://purl.org/dc/elements/1.1/">'
|
|
245
|
+
"<dc:title>t</dc:title><dc:language>en</dc:language>"
|
|
246
|
+
'<dc:identifier id="bid">pub-id</dc:identifier></metadata>'
|
|
247
|
+
"<manifest>"
|
|
248
|
+
'<item id="c1" href="d.xhtml" '
|
|
249
|
+
'media-type="application/xhtml+xml"/>'
|
|
250
|
+
'<item id="ncx" href="toc.ncx" '
|
|
251
|
+
'media-type="application/x-dtbncx+xml"/></manifest>'
|
|
252
|
+
'<spine toc="ncx"><itemref idref="c1"/></spine></package>',
|
|
253
|
+
)
|
|
254
|
+
z.writestr(
|
|
255
|
+
"toc.ncx",
|
|
256
|
+
'<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" '
|
|
257
|
+
'version="2005-1"><head>'
|
|
258
|
+
'<meta name="dtb:uid" content="pub-id"/>'
|
|
259
|
+
'<meta name="dtb:depth" content="1"/>'
|
|
260
|
+
'<meta name="dtb:totalPageCount" content="0"/>'
|
|
261
|
+
'<meta name="dtb:maxPageNumber" content="0"/></head>'
|
|
262
|
+
"<docTitle><text>t</text></docTitle><navMap>"
|
|
263
|
+
'<navPoint id="n1" playOrder="1"><navLabel><text>c</text>'
|
|
264
|
+
"</navLabel>"
|
|
265
|
+
'<content src="d.xhtml"/></navPoint></navMap></ncx>',
|
|
266
|
+
)
|
|
267
|
+
z.writestr(
|
|
268
|
+
"d.xhtml",
|
|
269
|
+
'<html xmlns="http://www.w3.org/1999/xhtml"><head><title>t'
|
|
270
|
+
"</title></head><body>"
|
|
271
|
+
'<p id="1:x">a</p><p id="2:x">b</p><p id="3:x">c</p>'
|
|
272
|
+
"<bogus>x</bogus>"
|
|
273
|
+
"</body></html>",
|
|
274
|
+
)
|
|
275
|
+
self.book = path
|
|
276
|
+
|
|
277
|
+
def tearDown(self) -> None:
|
|
278
|
+
self.tmp.cleanup()
|
|
279
|
+
|
|
280
|
+
def test_daemon_counts_match_the_subprocess_json_oracle(self) -> None:
|
|
281
|
+
import subprocess
|
|
282
|
+
|
|
283
|
+
d = _EpubcheckDaemon()
|
|
284
|
+
try:
|
|
285
|
+
daemon = d.check(self.book, timeout=180)
|
|
286
|
+
self.assertIsInstance(daemon, CheckResult)
|
|
287
|
+
out = subprocess.run(
|
|
288
|
+
["epubcheck", str(self.book), "--json", "-"],
|
|
289
|
+
capture_output=True,
|
|
290
|
+
text=True,
|
|
291
|
+
timeout=180,
|
|
292
|
+
env=_english_locale_env(),
|
|
293
|
+
check=False,
|
|
294
|
+
)
|
|
295
|
+
oracle = _counts_from_json(out.stdout)
|
|
296
|
+
self.assertEqual(
|
|
297
|
+
(daemon.fatals, daemon.errors, daemon.warnings),
|
|
298
|
+
(oracle.fatals, oracle.errors, oracle.warnings),
|
|
299
|
+
)
|
|
300
|
+
# the dedup pin: three identical id defects plus one different
|
|
301
|
+
# error are 4 occurrences but 2 aggregated messages, and the
|
|
302
|
+
# daemon reports the aggregated count, not the occurrences
|
|
303
|
+
self.assertEqual(daemon.errors, 2)
|
|
304
|
+
finally:
|
|
305
|
+
d.stop()
|
|
306
|
+
|
|
307
|
+
|
|
208
308
|
class DaemonPoolLogic(unittest.TestCase):
|
|
209
309
|
"""The pool's degrade-to-subprocess logic, hermetic (no JVM needed).
|
|
210
310
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|