bindery-cli 0.39.0__tar.gz → 0.41.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bindery_cli-0.41.0/.github/workflows/ci.yml +86 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/.github/workflows/publish.yml +26 -9
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/.gitignore +5 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/CLAUDE.md +28 -21
- bindery_cli-0.39.0/README.md → bindery_cli-0.41.0/PKG-INFO +106 -21
- bindery_cli-0.39.0/PKG-INFO → bindery_cli-0.41.0/README.md +79 -43
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/patchnotes.md +171 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/plugin/__init__.py +43 -6
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/pyproject.toml +24 -4
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/roadmap.md +247 -8
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/scripts/FastSweep.java +41 -2
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/scripts/fast_sweep.py +3 -2
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/spec.md +91 -30
- bindery_cli-0.41.0/src/bindery/__init__.py +19 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/audit.py +116 -49
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/cli.py +270 -18
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/epub.py +328 -21
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/library.py +33 -3
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/pagination.py +11 -6
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/transforms.py +20 -10
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/validate.py +40 -17
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/watermark.py +2 -1
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_audit.py +227 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_cli.py +319 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_epub.py +226 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_library.py +66 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_plugin.py +25 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_transforms.py +9 -0
- bindery_cli-0.41.0/tests/test_version.py +78 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/uv.lock +16 -12
- bindery_cli-0.39.0/.github/workflows/ci.yml +0 -25
- bindery_cli-0.39.0/REPORT-12-Sept.md +0 -119
- bindery_cli-0.39.0/src/bindery/__init__.py +0 -12
- bindery_cli-0.39.0/testing_facility/top500candidates/REPORT.md +0 -20
- bindery_cli-0.39.0/tests/test_version.py +0 -39
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/AGENTS.md +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/LICENSE +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/docs/screenshots/library-sweep.png +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/logo.svg +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/run_tests.sh +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/scripts/build_plugin.py +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/__main__.py +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/src/bindery/reserialize.py +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/testing_facility/books_to_fix/.gitkeep +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_pagination.py +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_reserialize.py +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_validate.py +0 -0
- {bindery_cli-0.39.0 → bindery_cli-0.41.0}/tests/test_watermark.py +0 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main]
|
|
5
|
+
pull_request:
|
|
6
|
+
# A push supersedes the run for the same ref: the tree it tested is gone.
|
|
7
|
+
concurrency:
|
|
8
|
+
group: ci-${{ github.ref }}
|
|
9
|
+
cancel-in-progress: true
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
jobs:
|
|
13
|
+
lint:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v5
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: "3.14"
|
|
20
|
+
# ruff is a dev-only linter; the rule set lives in pyproject.toml, so
|
|
21
|
+
# these mean the same thing locally. Checking `.` rather than named
|
|
22
|
+
# directories keeps a new top-level script from silently escaping lint.
|
|
23
|
+
- run: pip install ruff==0.16.2
|
|
24
|
+
- run: ruff check .
|
|
25
|
+
- run: ruff format --check .
|
|
26
|
+
test:
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@v5
|
|
30
|
+
- uses: actions/setup-python@v5
|
|
31
|
+
with:
|
|
32
|
+
python-version: "3.14"
|
|
33
|
+
# html5lib is the project's declared optional `reserialize` extra (core
|
|
34
|
+
# stays stdlib-only); installing it here lets CI exercise the
|
|
35
|
+
# --reserialize tests instead of skipping them.
|
|
36
|
+
- run: pip install . html5lib
|
|
37
|
+
- run: PYTHONPATH=src python -m unittest discover -s tests
|
|
38
|
+
core-compat:
|
|
39
|
+
name: Run the stack-free suite on the 3.12/3.13 floor
|
|
40
|
+
runs-on: ubuntu-latest
|
|
41
|
+
strategy:
|
|
42
|
+
fail-fast: false
|
|
43
|
+
matrix:
|
|
44
|
+
# The install floor (decision 63): vir-tui/cquarry are marker-gated
|
|
45
|
+
# to 3.14 there, so these legs install the core (tqdm only) and run
|
|
46
|
+
# exactly the test modules that never touch the gated stack: the
|
|
47
|
+
# repair core, the plugin under calibre stubs, validate, and the
|
|
48
|
+
# version/grammar pins. audit/library/CLI-stack tests stay on the
|
|
49
|
+
# 3.14 leg above.
|
|
50
|
+
python-version: ["3.12", "3.13"]
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v5
|
|
53
|
+
- uses: actions/setup-python@v5
|
|
54
|
+
with:
|
|
55
|
+
python-version: ${{ matrix.python-version }}
|
|
56
|
+
- run: pip install . html5lib
|
|
57
|
+
- run: >
|
|
58
|
+
PYTHONPATH=src python -m unittest
|
|
59
|
+
tests.test_transforms tests.test_epub tests.test_pagination
|
|
60
|
+
tests.test_watermark tests.test_reserialize tests.test_version
|
|
61
|
+
tests.test_plugin tests.test_validate
|
|
62
|
+
plugin-compat:
|
|
63
|
+
name: Parse-check the plugin zip under the interpreters Calibre embeds
|
|
64
|
+
runs-on: ubuntu-latest
|
|
65
|
+
strategy:
|
|
66
|
+
fail-fast: false
|
|
67
|
+
matrix:
|
|
68
|
+
# v0.39.0 shipped 3.14-only syntax and the plugin failed to load with
|
|
69
|
+
# a bare SyntaxError on every released Calibre. The embedded
|
|
70
|
+
# interpreters are 3.11 (7.x and the whole 8.x series) and 3.14
|
|
71
|
+
# (9.0+); 3.12/3.13 bracket the gap and match the CLI's grammar
|
|
72
|
+
# floor (tests/test_version.py). The plugin cannot be imported
|
|
73
|
+
# without Calibre, but the shipped zip can be extracted and
|
|
74
|
+
# byte-compiled, which catches exactly this class.
|
|
75
|
+
python-version: ["3.11", "3.12", "3.13", "3.14"]
|
|
76
|
+
steps:
|
|
77
|
+
- uses: actions/checkout@v5
|
|
78
|
+
- uses: actions/setup-python@v5
|
|
79
|
+
with:
|
|
80
|
+
python-version: ${{ matrix.python-version }}
|
|
81
|
+
- name: Build the plugin zip from this tree
|
|
82
|
+
run: python scripts/build_plugin.py
|
|
83
|
+
- name: Byte-compile the zip contents and the CLI sources
|
|
84
|
+
run: |
|
|
85
|
+
python -m zipfile -e dist/BinderyRepair-*.zip plugin_pkg
|
|
86
|
+
python -m compileall -q plugin_pkg src
|
|
@@ -5,11 +5,20 @@ on:
|
|
|
5
5
|
tags:
|
|
6
6
|
- "v*.*.*"
|
|
7
7
|
|
|
8
|
+
# A tag push is a release: never cancel a publish mid-run, and two tags
|
|
9
|
+
# publishing concurrently would race the release page; serialize instead.
|
|
10
|
+
concurrency:
|
|
11
|
+
group: publish-${{ github.ref }}
|
|
12
|
+
cancel-in-progress: false
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: read
|
|
16
|
+
|
|
8
17
|
jobs:
|
|
9
18
|
test:
|
|
10
19
|
runs-on: ubuntu-latest
|
|
11
20
|
steps:
|
|
12
|
-
- uses: actions/checkout@
|
|
21
|
+
- uses: actions/checkout@v5
|
|
13
22
|
- uses: actions/setup-python@v5
|
|
14
23
|
with:
|
|
15
24
|
python-version: "3.14"
|
|
@@ -30,7 +39,7 @@ jobs:
|
|
|
30
39
|
id-token: write # Required for OIDC authentication with PyPI
|
|
31
40
|
|
|
32
41
|
steps:
|
|
33
|
-
- uses: actions/checkout@
|
|
42
|
+
- uses: actions/checkout@v5
|
|
34
43
|
- uses: actions/setup-python@v5
|
|
35
44
|
with:
|
|
36
45
|
python-version: "3.14"
|
|
@@ -51,20 +60,28 @@ jobs:
|
|
|
51
60
|
permissions:
|
|
52
61
|
contents: write # create the release and attach the plugin zip
|
|
53
62
|
steps:
|
|
54
|
-
- uses: actions/checkout@
|
|
63
|
+
- uses: actions/checkout@v5
|
|
55
64
|
- uses: actions/setup-python@v5
|
|
56
65
|
with:
|
|
57
66
|
python-version: "3.14"
|
|
58
67
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
PYTHONPATH=src python -m unittest discover -s tests
|
|
63
|
-
|
|
68
|
+
# The suite already ran in the `test` job for this exact commit
|
|
69
|
+
# (needs: gates this job on it); running it again per job doubled the
|
|
70
|
+
# publish wall time for no extra signal.
|
|
64
71
|
- name: Build the plugin zip from the tagged tree
|
|
65
72
|
run: python scripts/build_plugin.py
|
|
66
73
|
|
|
74
|
+
# The tag message IS the release's patchnotes entry (verbatim, per the
|
|
75
|
+
# tagging procedure), so the release body is extracted from the tag
|
|
76
|
+
# object rather than paraphrased here. Without it the release page
|
|
77
|
+
# renders blank beside the zip.
|
|
78
|
+
- name: Extract the release body from the tag message
|
|
79
|
+
run: git tag -l --format='%(contents)' "${GITHUB_REF_NAME}" > release-body.md
|
|
80
|
+
|
|
81
|
+
# SHA-pinned to action-gh-release v2 (3bb1273): it carries contents:
|
|
82
|
+
# write, so the third-party action runs pinned, not floating.
|
|
67
83
|
- name: Attach BinderyRepair zip to the release
|
|
68
|
-
uses: softprops/action-gh-release@
|
|
84
|
+
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65
|
|
69
85
|
with:
|
|
86
|
+
body_path: release-body.md
|
|
70
87
|
files: dist/BinderyRepair-*.zip
|
|
@@ -12,6 +12,11 @@ testing_facility/books_to_fix/*
|
|
|
12
12
|
!testing_facility/books_to_fix/.gitkeep
|
|
13
13
|
testing_facility/top500candidates/*.epub
|
|
14
14
|
|
|
15
|
+
# The census report lists real titles at absolute library paths; the same
|
|
16
|
+
# leak class as the book files above. Untracked 2026-09-15 (forward-only;
|
|
17
|
+
# the file stays on disk as the --summary target).
|
|
18
|
+
testing_facility/top500candidates/REPORT.md
|
|
19
|
+
|
|
15
20
|
# test_facility EPUBs stopped being tracked in v0.33.0 (forward-only; 9 of
|
|
16
21
|
# the 10 books are commercial and one untracked leak on a public repo is
|
|
17
22
|
# one too many). Files stay on disk, out of git.
|
|
@@ -15,7 +15,11 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
|
|
|
15
15
|
- **Minimal Dependencies.** Runtime deps are exactly: `tqdm` (progress/output),
|
|
16
16
|
`vir-tui` (shared TUI rendering) and `cquarry` (read-only Calibre `metadata.db` access,
|
|
17
17
|
adopted in v0.16.0), pinned as PyPI ranges in pyproject.toml (`vir-tui>=2.2.0`,
|
|
18
|
-
`cquarry>=1.
|
|
18
|
+
`cquarry>=1.19.0`); `uv.lock` records the resolved versions. Since v0.40.0 both
|
|
19
|
+
VirInvictus pins carry `; python_version >= '3.14'` markers (every release they
|
|
20
|
+
ship declares requires-python >=3.14) and the package floor is 3.12: a 3.12/3.13
|
|
21
|
+
install runs the stack-free repair core, and the audit/library surfaces degrade
|
|
22
|
+
through cli.py's dispatch guard instead of failing resolution. `html5lib` remains the one approved
|
|
19
23
|
heavy-parsing exception (used only for the `--reserialize` fix, imported lazily so every
|
|
20
24
|
other mode runs without it). Tests use the standard `unittest` framework. epubcheck is an
|
|
21
25
|
external CLI dependency expected on PATH; `--install-to-calibre` needs no external
|
|
@@ -23,9 +27,11 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
|
|
|
23
27
|
subprocess was retired in v0.24.0). Before adding any further Python package, stop and ask.
|
|
24
28
|
- **Semantics-preserving transforms by default, everything else fenced behind a flag.**
|
|
25
29
|
The always-on core is exactly five well-formedness fixes (prolog junk, duplicate
|
|
26
|
-
`xmlns`, bare `&`, named entities, void self-closing)
|
|
30
|
+
`xmlns`, bare `&`, named entities, void self-closing), the NCX pipeline, and the
|
|
31
|
+
mimetype fix (added/normalized/first-stored, epub.py's archive rewrite); every core
|
|
27
32
|
fix must render identically to the author's intent: never add, remove, or reorder
|
|
28
|
-
visible content.
|
|
33
|
+
visible content. The canonical opt-in inventory lives in spec.md: fifteen structural
|
|
34
|
+
repairs, three lossy strips, four safe opt-ins, plus `--reserialize`.
|
|
29
35
|
* **Structural repairs** (`--fix-empty-body`, `--fix-missing-title`, `--fix-id-colons`,
|
|
30
36
|
`--fix-page-map`, `--strip-epub3-attrs`, `--downgrade-epub3-tags`,
|
|
31
37
|
`--unwrap-block-in-inline`, `--strip-invalid-value`, `--unwrap-illegal-tags`,
|
|
@@ -43,23 +49,23 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
|
|
|
43
49
|
`transforms.strip_attrs_in_start_tags` (prose mentioning the attributes and
|
|
44
50
|
CDATA/comment content are preserved), and `strip_invalid_value` matches the
|
|
45
51
|
attribute name with a `(?<![\w:.-])value` lookbehind so `data-value` survives.
|
|
46
|
-
* **Lossy strips** (`--strip-pagination`, `--strip-broken-tags`,
|
|
47
|
-
pagination.py, watermark.py
|
|
48
|
-
|
|
52
|
+
* **Lossy strips** (`--strip-pagination`, `--strip-broken-tags`,
|
|
53
|
+
`--strip-watermarks`; pagination.py, watermark.py; `--strip-stub-docs`,
|
|
54
|
+
epub.py's `detect_stub_docs`): remove only what a converter injected (page numbers,
|
|
55
|
+
running headers, leaked tags, watermarks, repeated placeholder chapters), fenced behind
|
|
56
|
+
character conservation, tag
|
|
49
57
|
balance, and the epubcheck no-regression bar. The watermark anchored pass's
|
|
50
58
|
whole-match fallback only fires when the match holds nothing but the stamp; a
|
|
51
59
|
larger match (an unclosed stamp anchor that swallowed prose) is refused, counted
|
|
52
60
|
in `RepairReport.watermark_refusals`, and surfaced as a `manual_watermark_repair`
|
|
53
|
-
decision by `run phase1` on both the read-only and apply paths.
|
|
61
|
+
decision by `run phase1` on both the read-only and apply paths. `--strip-stub-docs`
|
|
62
|
+
(2026-09-15) mirrors emptytext's placeholder signals: identical short text across
|
|
63
|
+
>= 3 spine docs and >= 30% of the spine; the whole-spine-stubs book is refused
|
|
64
|
+
(EMPTY, re-source), and the drop cascades to archive entries, manifest, spine,
|
|
65
|
+
NCX navPoints, and nav toc entries.
|
|
54
66
|
Do not let any NEW fix touch content without its own flag; if a candidate repair cannot
|
|
55
|
-
be made deterministically safe, it does not belong here
|
|
56
|
-
instead.
|
|
57
|
-
the safe opt-ins (`--fix-ids`, `--add-img-alt`, `--strip-bad-attrs`,
|
|
58
|
-
`--escape-unknown-entities`) plus `--encode-url-spaces`, which repair without
|
|
59
|
-
altering visible markup; the "twelve structural repairs and three lossy strips"
|
|
60
|
-
phrasing elsewhere in these docs was never the complete inventory, and the
|
|
61
|
-
Phase 16 package repairs (`--fix-container`, `--fix-media-types`, `--fix-cover`)
|
|
62
|
-
took the structural count to fifteen (v0.38.0 + v0.39.0).
|
|
67
|
+
be made deterministically safe, it does not belong here: report it for manual repair
|
|
68
|
+
instead.
|
|
63
69
|
- **The gate is the safety contract.** Never apply a repair epubcheck has not accepted.
|
|
64
70
|
Respect the two-mode logic in `validate.gate` (fatal-fixing tolerates error unmasking;
|
|
65
71
|
error-cleanup does not). The lossy strips (`--strip-pagination`, `--strip-broken-tags`,
|
|
@@ -80,17 +86,18 @@ Born from the 2026 library audit (see the user memory `calibre-library-epubcheck
|
|
|
80
86
|
- `src/bindery/pagination.py`: the opt-in lossy page-number strip (runhead detection, page-layer decision, block-centric removal/merge, safety nets).
|
|
81
87
|
- `src/bindery/watermark.py`: the opt-in lossy watermark strip (anchored and anchorless signature removal).
|
|
82
88
|
- `src/bindery/reserialize.py`: structural repair via `html5lib`.
|
|
83
|
-
- `src/bindery/audit.py`: read-only body-text audits (`content`, `pagenumbers`, `emptytext`, `ocr`, `monolithic` since v0.21.0 with `--max-doc-chars N`, and `completeness` since v0.36.0) behind the `audit` subcommand (v0.15.0). Since v0.18.0 library mode resolves EPUB paths through `cquarry.get_format_path()` and can apply a tag to flagged books via `cquarry.write.WritableCalibreDB` (only with explicit `--tag`; the only sanctioned write path). Since v0.19.0 `audit --id BOOK_ID` audits a single library book through cquarry's single-entity `get_book()` fetch (no library-wide cache; supports `--tag`; incompatible with the directory argument; comma lists since v0.23.0). Since v0.22.0 every archive entry is fully read (CRC + decompression) before analysis
|
|
84
|
-
- `src/bindery/epub.py`: archive rewrite, NCX uid sync, RepairReport, mismatch detection, and the opt-in structural-repair plumbing (including the CSS precondition scan). Since v0.38.0/v0.39.0 (Phase 16): `generate_container` + the `--fix-container` gateway (insert after mimetype when missing, replace in-loop when stale; constant-epoch timestamp keeps bytes deterministic), `prune_dangling_edges` (prune_missing_manifest_items returns the pruned ids and the edges that pointed at them
|
|
89
|
+
- `src/bindery/audit.py`: read-only body-text audits (`content`, `pagenumbers`, `emptytext`, `ocr`, `monolithic` since v0.21.0 with `--max-doc-chars N`, and `completeness` since v0.36.0) behind the `audit` subcommand (v0.15.0). Since v0.18.0 library mode resolves EPUB paths through `cquarry.get_format_path()` and can apply a tag to flagged books via `cquarry.write.WritableCalibreDB` (only with explicit `--tag`; the only sanctioned write path). Since v0.19.0 `audit --id BOOK_ID` audits a single library book through cquarry's single-entity `get_book()` fetch (no library-wide cache; supports `--tag`; incompatible with the directory argument; comma lists since v0.23.0). Since v0.22.0 every archive entry is fully read (CRC + decompression) before analysis (a damaged entry reports its own CORRUPT verdict instead of feeding emptytext) and manifest/NCX references to absent files classify as `convention` (bloated ToC, consecutive span) or `fragment` (broken span). Hazard: the scan loop reuses `tag` as its per-book display column, so the `--tag` argument is captured as `audit_tag` at function entry; do not collapse them again. Since v0.29.0 `audit --json FILE` writes per-file analyzer verdicts in the `library --json` shape (directory, library, and single-book modes; with `--id`, exactly one id; emptytext omitted from a record when the archive verdict owns the body-text story). Since v0.36.0 the completeness analyzer (Phase 15) reports the phase-1 spot-check per book: spine/prose-doc counts (prose = >= 400 visible chars), first/middle/last prose-doc opening/closing excerpts, trailing-ToC classification (`_trailing_toc`: short link lines dominate, no paragraph-length blocks), and the unreadable fraction. It is ADVISORY by contract (problem always False; ADVISORY on a trailing ToC or unreadable fraction >= 10%), never moves the exit code, and is skipped like emptytext when the archive verdict owns the book. Since v0.38.0 the archive verdict distinguishes font obfuscation from DRM (`OBFUSCATION_ALGOS`: IDPF 2008/embedding + both Adobe URIs; readable obfuscated entries are the benign OBFUSCATED advisory, an unreadable one is CORRUPT, and only non-obfuscation algorithms give ENCRYPTED). Since v0.40.0 the vir-tui console renderer loads through the module-level `_LazyUI` proxy (a plain `import vir_tui as ui` at module level would make 3.12/3.13 installs unimportable now that the stack is marker-gated; LOAD_GLOBAL never consults module `__getattr__`, so a proxy object is the lazy shape that works).
|
|
90
|
+
- `src/bindery/epub.py`: archive rewrite, NCX uid sync, RepairReport, mismatch detection, and the opt-in structural-repair plumbing (including the CSS precondition scan). Since v0.38.0/v0.39.0 (Phase 16): `generate_container` + the `--fix-container` gateway (insert after mimetype when missing, replace in-loop when stale; constant-epoch timestamp keeps bytes deterministic), `prune_dangling_edges` (prune_missing_manifest_items returns the pruned ids and the edges that pointed at them (spine@toc, media-overlay, fallback, EPUB2 cover meta) are rewritten so pruning cannot manufacture the regression the gate rejects), and `fix_manifest_media_types` (extension map + magic-byte confirmation via a `peek` callable over the open source zip; jpg/png/gif only), and `fix_cover_meta` (the ruling's deterministic half: dangling EPUB2 cover meta re-pointed from the guide's cover reference when exactly one manifest item carries that file, removed otherwise; accepted under no_worse like the lossy strips because cover wiring is invisible to epubcheck).
|
|
85
91
|
- `src/bindery/validate.py`: epubcheck wrapper, the `gate` (improvement) and `no_worse` (no-regression, for the lossy strips) acceptance bars.
|
|
86
|
-
- `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching); the `(id)` directory-name regex is only the no-catalog fallback. Since v0.24.0 `install_format()` places the repaired file atomically and updates the `data` row through `cquarry.write.WritableCalibreDB` (since v0.31.0 via `set_format`, cquarry 1.17's sanctioned remove+add in one transaction); the external `calibredb` CLI is gone. Since v0.32.0 a guessed id drives a row update only when metadata.db corroborates it (`_verify_guess`: books row exists, file in the book's own directory, catalogued `data.name` when a row exists); anything else saves in place and leaves the catalog untouched
|
|
87
|
-
- `src/bindery/cli.py`: `repair` and `library` subcommands, including `--all` and `--install-to-calibre`; since v0.29.0 also the `run` verbs: `phase1` (pre-import vetting over loose files: the audit battery composed with one gated repair sweep; read-only until `--apply-lossy`, which IS the lossy-strip consent) and `phase3` (the scoped post-import apply step: `library --id --sweep --only all --apply --all --install-to-calibre` plus a pre/post summary; mechanically refuses unscoped sweeps with exit 2). Both take `--non-interactive` and surface open questions as `decisions_needed` in their JSON, never prompts. They drive the shipped `run_library` through the real parser, so no flag can drift between the verb and the subcommand it wraps. `library --workers N` (v0.30.0) parallelizes only the sweep's candidate pass, in windows of N consumed in order so `--limit` stays lazy; the repair phase is never parallel (shared workdir, atomic-replacement contract).
|
|
88
|
-
- `plugin/__init__.py` + `scripts/build_plugin.py`: the Bindery Repair Calibre plugin (v0.37.0; identity `Bindery Repair` / `bindery_repair`, approved and built per the scoped spec in roadmap.md Phase 3). The builder vendors `transforms.py`, `epub.py`, `pagination.py`, `watermark.py`, `reserialize.py` byte-identically into the zip (the zip root is a package, so their relative imports resolve unchanged; the suite's drift test pins the equality) with the version tuple substituted from the single-source VERSION; `publish.yml` attaches `BinderyRepair-v<VERSION>.zip` to each release. The plugin runs ONLY the default pass (epubcheck cannot gate inside Calibre), never raises, is byte-idempotent (zero fixes returns the original path), testzip()-verifies its output, logs one line per book, and takes JSON config via `site_customization` (`log`, `log_path`, `max_size_mb` default 150, `epubcheck_path` experimental validation default OFF). Load it in tests through the `tests/test_plugin.py` stubs: the calibre namespace is installed in sys.modules and the package loads with a two-dir `__path__` (plugin dir first, then src/bindery) exactly like calibre's zipplugin loader shape.
|
|
89
|
-
- `tests/`: transforms, end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic/completeness, archive integrity, spine classification), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, the version pin, and the Calibre plugin (build drift + run behavior under calibre stubs).
|
|
92
|
+
- `src/bindery/library.py`: Calibre walk, atomic replace, backups, and native format installation. Since v0.19.0 `CalibreIdResolver` resolves the book id from `metadata.db` through cquarry (one lazy path→id map per run; since v0.20.0 the map comes from `CalibreDB.format_path_index()`, re-normalized with `resolve().lower()` for the resolver's symlinked-directory and case-insensitive matching); the `(id)` directory-name regex is only the no-catalog fallback. Since v0.24.0 `install_format()` places the repaired file atomically and updates the `data` row through `cquarry.write.WritableCalibreDB` (since v0.31.0 via `set_format`, cquarry 1.17's sanctioned remove+add in one transaction); the external `calibredb` CLI is gone. Since v0.32.0 a guessed id drives a row update only when metadata.db corroborates it (`_verify_guess`: books row exists, file in the book's own directory, catalogued `data.name` when a row exists); anything else saves in place and leaves the catalog untouched: never reintroduce directory-name guessing as the primary source: renamed/mismatched directories would replace the wrong book. Since 2026-09-15 `make_backup` takes the opt-in `--backup-keep N` ring: at most N backup files per book, the author original `.bak` never deleted, newest content at the highest name; default remains an unbounded rotation.
|
|
93
|
+
- `src/bindery/cli.py`: `repair` and `library` subcommands, including `--all` and `--install-to-calibre`; `repair --json` emits the library per-book record vocabulary (2026-09-15); since v0.29.0 also the `run` verbs: `phase1` (pre-import vetting over loose files: the audit battery composed with one gated repair sweep; read-only until `--apply-lossy`, which IS the lossy-strip consent) and `phase3` (the scoped post-import apply step: `library --id --sweep --only all --apply --all --install-to-calibre` plus a pre/post summary; mechanically refuses unscoped sweeps with exit 2; since 2026-09-15 the summary sums applied/equal states only and carries rejected projections as `rejected_projection`). Plus `bindery doctor` (2026-09-15): the environment self-check that imports none of the VirInvictus stack, never raises, and always exits 0. Both run verbs take `--non-interactive` and surface open questions as `decisions_needed` in their JSON, never prompts. They drive the shipped `run_library` through the real parser, so no flag can drift between the verb and the subcommand it wraps. `library --workers N` (v0.30.0) parallelizes only the sweep's candidate pass, in windows of N consumed in order so `--limit` stays lazy; the repair phase is never parallel (shared workdir, atomic-replacement contract). Since v0.40.0 `main()` guards the VirInvictus stack: a `ModuleNotFoundError` naming `vir_tui`/`cquarry` (marker-gated to 3.14+ on the 3.12 floor) becomes one stderr line and trouble exit 2 instead of a traceback; anything else missing still raises.
|
|
94
|
+
- `plugin/__init__.py` + `scripts/build_plugin.py`: the Bindery Repair Calibre plugin (v0.37.0; identity `Bindery Repair` / `bindery_repair`, approved and built per the scoped spec in roadmap.md Phase 3). The builder vendors `transforms.py`, `epub.py`, `pagination.py`, `watermark.py`, `reserialize.py` byte-identically into the zip (the zip root is a package, so their relative imports resolve unchanged; the suite's drift test pins the equality) with the version tuple substituted from the single-source VERSION; `publish.yml` attaches `BinderyRepair-v<VERSION>.zip` to each release. The plugin runs ONLY the default pass (epubcheck cannot gate inside Calibre), never raises, is byte-idempotent (zero fixes returns the original path), testzip()-verifies its output, logs one line per book, and takes JSON config via `site_customization` (`log`, `log_path`, `max_size_mb` default 150, `epubcheck_path` experimental validation default OFF). Load it in tests through the `tests/test_plugin.py` stubs: the calibre namespace is installed in sys.modules and the package loads with a two-dir `__path__` (plugin dir first, then src/bindery) exactly like calibre's zipplugin loader shape. **Plugin/interpreter compatibility (v0.40.0):** v0.39.0 shipped PEP 758 unparenthesized except tuples, and the plugin died with a bare SyntaxError on every released Calibre (7.x and the whole 8.x series embed Python 3.11, 9.0+ embeds 3.14; from calibre's bypy sources.json, checked against the upstream clone). The core is now grammar-pinned: parenthesized except tuples everywhere, `ruff`'s `target-version = "py312"` pin (inference from requires-python would rewrite the parens back off), `tests/test_version.py`'s parse guard, and CI's `plugin-compat` job byte-compiling the built zip under 3.11/3.12/3.13/3.14. `minimum_calibre_version` is (7, 0, 0): the oldest series whose interpreter CI covers, replacing the hollow (2, 0, 0); `supported_platforms` stays `["linux"]` (developed and tested on Linux only). Any new syntax in the vendored modules must clear the guard before it ships.
|
|
95
|
+
- `tests/`: transforms, end-to-end repair, atomic replace, pagination, watermarks, the audit analyzers (content/pagenumbers/emptytext/ocr/monolithic/completeness, archive integrity, spine classification), the library sweep and `CalibreIdResolver`, validate, reserialize, CLI wiring, the version pin and the syntax-floor grammar guard, and the Calibre plugin (build drift + run behavior under calibre stubs).
|
|
90
96
|
|
|
91
97
|
## Conventions
|
|
92
98
|
|
|
93
99
|
- Type hints, `from __future__ import annotations`, ruff for lint and format.
|
|
100
|
+
- `./run_tests.sh` runs the suite in the project venv. Since the 3.12 floor, `uv run --python X` REBUILDS `.venv` for interpreter X, and the `python_version >= '3.14'` markers then gate vir-tui/cquarry OUT of it below 3.14 (the suite goes red with ModuleNotFoundError: cquarry). After interpreter experiments, rebuild with `uv run --python 3.14` before running the suite again.
|
|
94
101
|
- `VERSION` lives in `src/bindery/__init__.py`, mirrored in `pyproject.toml`. Bump both.
|
|
95
102
|
- Run tests with `./run_tests.sh`.
|
|
96
103
|
|
|
@@ -1,3 +1,30 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: bindery-cli
|
|
3
|
+
Version: 0.41.0
|
|
4
|
+
Summary: Audit EPUB body text and repair broken EPUBs with deterministic, epubcheck-gated fixes; atomic in-place replacement in a Calibre library.
|
|
5
|
+
Project-URL: Homepage, https://github.com/VirInvictus/bindery-cli
|
|
6
|
+
Project-URL: Repository, https://github.com/VirInvictus/bindery-cli
|
|
7
|
+
Project-URL: Changelog, https://github.com/VirInvictus/bindery-cli/blob/main/patchnotes.md
|
|
8
|
+
Author: vrnvctss
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: calibre,ebook,epub,epubcheck,html-repair,repair,watermarks
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Topic :: Utilities
|
|
20
|
+
Requires-Python: >=3.12
|
|
21
|
+
Requires-Dist: cquarry>=1.19.0; python_version >= '3.14'
|
|
22
|
+
Requires-Dist: tqdm
|
|
23
|
+
Requires-Dist: vir-tui>=2.5.0; python_version >= '3.14'
|
|
24
|
+
Provides-Extra: reserialize
|
|
25
|
+
Requires-Dist: html5lib>=1.1; extra == 'reserialize'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
1
28
|
<div align="center">
|
|
2
29
|
<img src="logo.svg" width="96" height="96" alt="bindery-cli logo"/>
|
|
3
30
|
<h1>bindery-cli</h1>
|
|
@@ -10,7 +37,7 @@
|
|
|
10
37
|
|
|
11
38
|
## What it fixes
|
|
12
39
|
|
|
13
|
-
bindery-cli makes accidentally broken markup well-formed again. It does not rewrite or reflow content. The **default pass** applies only a small set of deterministic, semantics-preserving well-formedness fixes that real-world EPUBs (especially Calibre conversions) trip over
|
|
40
|
+
bindery-cli makes accidentally broken markup well-formed again. It does not rewrite or reflow content. The **default pass** applies only a small set of deterministic, semantics-preserving well-formedness fixes that real-world EPUBs (especially Calibre conversions) trip over; nothing else changes without an explicit flag:
|
|
14
41
|
|
|
15
42
|
- **Unclosed void elements** (`<link>`, `<br>`, `<img>`, ...) get self-closed.
|
|
16
43
|
- **Undeclared named entities** (` `, `°`, `é`, ...) become numeric character references that every XML parser understands.
|
|
@@ -22,7 +49,7 @@ bindery-cli makes accidentally broken markup well-formed again. It does not rewr
|
|
|
22
49
|
|
|
23
50
|
### Opt-in repairs
|
|
24
51
|
|
|
25
|
-
Everything below is off until its flag is passed (or all at once via `--all`). They go further than well-formedness
|
|
52
|
+
Everything below is off until its flag is passed (or all at once via `--all`). They go further than well-formedness (altering structure or adding minimal content), so they never run by default:
|
|
26
53
|
|
|
27
54
|
- **`--fix-ids`**: rewrite ids that are not valid XML names (start with a digit, contain a colon) in the OPF manifest, updating every reference to them (spine, fallback, media-overlay, the EPUB 2 cover meta), and in the NCX (where old conversions stamp navPoint ids from UUIDs). Touches the OPF, so it is off by default; the dc: metadata is never altered.
|
|
28
55
|
- **`--add-img-alt`**: add `alt=""` to `<img>` elements missing the required attribute. Renders identically, but it adds markup the author never wrote, and an empty alt tells a screen reader the image is decorative; hence opt-in.
|
|
@@ -34,7 +61,7 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
|
|
|
34
61
|
- **`--fix-missing-title`**: injects a `<title>Unknown</title>` fallback in the `<head>` if missing, handling both empty `<title/>` self-closing tags and entirely absent tags.
|
|
35
62
|
- **`--unwrap-block-in-inline`**: safely unwraps `<span>` tags that illegally contain a block-level element (e.g. `<div>` or `<p>`), leaving the block element intact.
|
|
36
63
|
- **`--strip-invalid-value`**: systematically strips invalid `value="..."` attributes from elements like `<div>`, `<span>`, `<p>`, etc.
|
|
37
|
-
- **`--fix-page-map`**: normalizes legacy page-map markup
|
|
64
|
+
- **`--fix-page-map`**: normalizes legacy page-map markup: drops the non-standard `page-map="..."` attribute from the OPF `<spine>` and adds `class="pages"` to classless NCX `<pageList>` elements (epubcheck rejects both on older HarperCollins / Anna's Archive conversions).
|
|
38
65
|
- **`--strip-epub3-attrs`**: scrubs the EPUB3-only attributes epubcheck rejects on an EPUB2 package: `page-progression-direction`, `epub:type`, `aria-label` (a fixed, documented set; rendering is unchanged, and lookalikes like `type` or the wider aria family survive). The edit is anchored to real start tags, so prose that merely mentions `epub:type="chapter"` is preserved.
|
|
39
66
|
- **`--downgrade-epub3-tags`**: downgrades EPUB3/HTML5 semantic elements to their EPUB2 equivalents (`figure`/`section` to `div`, `figcaption` to `p`), keeping existing classes and appending the semantic name (`class="figure"`) as the styling hook. Tag names a stylesheet styles as an element selector are protected book-wide, so styled formatting is never destroyed. Both EPUB2-targeted fixes are gated on the package version declared in the OPF: on an EPUB 3 book they do nothing, because their target defects only exist below EPUB 3.
|
|
40
67
|
- **`--unwrap-illegal-tags`**: strips completely invalid or deprecated HTML tags that break EPUB3 validation (`<st>`, `<sentence>`, `<o>`, `<w>`, `<pagebreak>`) while retaining their inner text. Any of those names styled as an *element selector* by an EPUB stylesheet (`.css` entries and inline `<style>` blocks alike; class/id selectors like `.st`/`#w` don't count) is protected for the whole book, guaranteeing format preservation.
|
|
@@ -42,15 +69,16 @@ Everything below is off until its flag is passed (or all at once via `--all`). T
|
|
|
42
69
|
- **`--strip-broken-anchors`**: strips href attributes that cannot resolve, keeping the anchor text byte-for-byte: a `#fragment` the target document does not define (RSC-020/RSC-012; NCX navTargets fall back to the document target, so chapter navigation survives) and unresolvable URI schemes (`kindle:`, `file:`).
|
|
43
70
|
- **`--encode-url-spaces`**: percent-encodes raw spaces in `src`/`href` attribute values across the package (OPF manifest, NCX, content documents). A URL with a literal space is not a valid URL (RSC-020) and unresolvable on strict readers; the encoded form denotes the same file.
|
|
44
71
|
- **`--fix-container`**: generates `META-INF/container.xml` at the located OPF when the container is missing or names a file the archive does not contain. This is the gateway defect: epubcheck stays fatal while the OPF is unfindable, so no repair can be gate-accepted on such a book until the container exists.
|
|
45
|
-
- **`--fix-media-types`**: normalizes wrong manifest media-type declarations (OPF-029, e.g. a jpg stamped `image/png` by an aggregator). Attribute-only and quote-preserving; it fires only when the file's magic bytes confirm the extension, so
|
|
72
|
+
- **`--fix-media-types`**: normalizes wrong manifest media-type declarations (OPF-029, e.g. a jpg stamped `image/png` by an aggregator). Attribute-only and quote-preserving; it fires only when the file's magic bytes confirm the extension, so a misnamed-but-consistent file (a PNG renamed .jpg) is never made worse.
|
|
46
73
|
- **`--fix-cover`**: repairs dangling EPUB2 cover wiring. A `<meta name="cover">` whose content names no manifest item is re-pointed when the OPF guide's own cover reference names an existing item, and removed when nothing identifies it; EPUB3 `properties="cover-image"` is deliberately audit-only (guessing which image is the cover is not deterministic). Cover wiring is invisible to epubcheck, so cover-only repairs are accepted under the same no-worse bar the lossy strips use.
|
|
47
74
|
|
|
48
75
|
|
|
49
|
-
|
|
76
|
+
Four opt-in fixes are **lossy** and stand apart from the semantics-preserving rest:
|
|
50
77
|
|
|
51
78
|
- **`--strip-pagination`**: remove print page numbers and running headers that a PDF/OCR conversion baked into the body text as literal paragraphs (so they reflow into the middle of a sentence: "where the hay cart **16** was taking him"). It removes only that injected furniture, never the author's prose: where a number split a sentence it rejoins the two paragraphs (closing up a word split like `compli-` / `mentary`), and it preserves roman chapter numbers, page-list nav anchors, and years. A roman numeral counts as a page number only when it is well-formed and under 100, so ordinary words (`mid`, `mix`, `civil`) are never mistaken for pages. A book is only treated as paginated when it has both a dense run of arabic numbers and several confident mid-sentence interrupts, so a merely chapter-numbered book is left alone. Three safety nets guard every edit (character conservation, tag balance, and an epubcheck no-regression check); any failure leaves the document untouched.
|
|
52
79
|
- **`--strip-broken-tags`**: remove leaked HTML closing tags missing their open brackets (e.g. `</p>`) that render as raw text.
|
|
53
80
|
- **`--strip-watermarks`**: remove known producer and distributor watermarks (e.g. OceanofPDF, ABC Amber LIT Converter) and stray marker files. It locates the stamp and deletes the outermost wrapper whose entire visible text is the watermark, ensuring prose that merely mentions the URL is preserved. An inline stamp link is deleted only when it holds nothing but the stamp; a match too large to be safe (an unclosed stamp anchor that swallowed prose) is refused and reported for manual repair rather than deleted.
|
|
81
|
+
- **`--strip-stub-docs`**: drop spine documents whose entire visible text is one identical short placeholder repeated across the spine: the Bookmate-style export whose chapters are all the same "content unavailable" notice, passing epubcheck because the placeholder is valid XHTML. The drop cascades (archive entries, manifest items, spine order, NCX navPoints, nav toc entries), so the book opens straight into its real chapters. The identity rule is deliberately conservative (same text, at least 3 docs, at least 30% of the spine, each doc short but not blank) and a book whose *every* spine doc is the stub is refused outright: that book is empty and needs a re-source, never a repair.
|
|
54
82
|
|
|
55
83
|
All lossy edits are invisible to epubcheck, so they are accepted when the result is *no worse* rather than measurably better.
|
|
56
84
|
|
|
@@ -69,7 +97,39 @@ The lossy modes (`--strip-pagination`, `--strip-broken-tags`, and `--strip-water
|
|
|
69
97
|
|
|
70
98
|
## Install
|
|
71
99
|
|
|
72
|
-
Python 3.
|
|
100
|
+
Python 3.12+, plus epubcheck on `PATH` for the gate. Dependencies: `tqdm` for
|
|
101
|
+
progress output, plus the VirInvictus libraries `vir-tui` (TUI rendering) and
|
|
102
|
+
`cquarry` (read-only Calibre database access), which install on Python 3.14+
|
|
103
|
+
only; `html5lib` remains an optional extra, needed only for `--reserialize`.
|
|
104
|
+
|
|
105
|
+
epubcheck (W3C's conformance checker) is the gate's oracle and the one
|
|
106
|
+
dependency pip cannot install for you. It needs a Java runtime. Any of these
|
|
107
|
+
works:
|
|
108
|
+
|
|
109
|
+
- your package manager, where it ships: `apt install epubcheck` (Debian/Ubuntu),
|
|
110
|
+
`brew install epubcheck` (macOS), `epubcheck` in the AUR (Arch); or
|
|
111
|
+
- the release zip from [w3c/epubcheck](https://github.com/w3c/epubcheck/releases):
|
|
112
|
+
unzip it and put its `epubcheck` launcher script somewhere on `PATH`.
|
|
113
|
+
|
|
114
|
+
Verify with `epubcheck --version`. A validated run refuses to start without it
|
|
115
|
+
(exit 1: "epubcheck not found") rather than silently skipping the gate; only
|
|
116
|
+
`--no-validate` bypasses it, and the Calibre plugin never needs it (its default
|
|
117
|
+
pass runs ungated by design). `scripts/fast_sweep.py` also accepts
|
|
118
|
+
`EPUBCHECK_JAR` pointing at the jar directly.
|
|
119
|
+
|
|
120
|
+
`bindery doctor` runs all of these checks for you: it reports the Python
|
|
121
|
+
stack tier, the epubcheck version, Java, the optional html5lib, and whether a
|
|
122
|
+
Calibre library is discoverable from the current directory. It always works
|
|
123
|
+
and always exits 0; the findings are the output.
|
|
124
|
+
|
|
125
|
+
The floor is two-tier by those markers: on Python 3.14 everything runs. On
|
|
126
|
+
3.12/3.13 the install carries the repair core only, which is the tool's core
|
|
127
|
+
job: `bindery repair` on a single EPUB works in full, while `bindery audit`,
|
|
128
|
+
`bindery library` and `bindery run` print a one-line note that the command
|
|
129
|
+
needs the Python 3.14 stack instead of dying in a traceback. (Every vir-tui
|
|
130
|
+
and cquarry release on PyPI currently declares `requires-python >=3.14`, so
|
|
131
|
+
the stack itself cannot follow the floor down until they move.) CI runs the
|
|
132
|
+
stack-free test modules on real 3.12 and 3.13 interpreters.
|
|
73
133
|
|
|
74
134
|
```sh
|
|
75
135
|
uv tool install bindery-cli # core tools
|
|
@@ -82,10 +142,23 @@ Each release also ships **BinderyRepair-v<VERSION>.zip**, a Calibre plugin that
|
|
|
82
142
|
repairs EPUBs as they are imported: it is attached to the GitHub release beside
|
|
83
143
|
the PyPI wheel (install it via Calibre's Preferences, Plugins, "Load plugin
|
|
84
144
|
from file"). The plugin is the CLI's always-on core pass only: the five
|
|
85
|
-
well-formedness fixes
|
|
145
|
+
well-formedness fixes, the NCX pipeline, and the mimetype fix. The structural
|
|
146
|
+
repairs and lossy
|
|
86
147
|
strips stay CLI-only, because their acceptance is the epubcheck gate and
|
|
87
148
|
epubcheck cannot run inside Calibre; nothing runs ungated in the plugin, ever.
|
|
88
149
|
|
|
150
|
+
Compatibility: the plugin declares a minimum Calibre of **7.0** and
|
|
151
|
+
`supported_platforms` is Linux. Both floors state what is actually verified, no more:
|
|
152
|
+
released Calibre embeds Python 3.11 through the whole 8.x series and 3.14 from
|
|
153
|
+
9.0 on, and CI byte-compiles the shipped plugin zip under Python 3.11, 3.12,
|
|
154
|
+
3.13 and 3.14, so the plugin parses on every current Calibre series; 7.x is
|
|
155
|
+
the oldest whose interpreter CI covers, 2.x-6.x embed interpreters nothing
|
|
156
|
+
verifies. The Calibre API surface the plugin uses is the long-stable
|
|
157
|
+
`FileTypePlugin` shape, exercised against test stubs only; the plugin is
|
|
158
|
+
developed and tested on Linux (v0.39.0 shipped the 3.14-only except syntax
|
|
159
|
+
and silently failed to load on every released Calibre, which is the failure
|
|
160
|
+
this floor and CI now guard against).
|
|
161
|
+
|
|
89
162
|
Behavior and guarantees:
|
|
90
163
|
|
|
91
164
|
- `run()` never raises and never touches the original file or `metadata.db`: a
|
|
@@ -103,9 +176,12 @@ Configuration is the plugin's customization string, parsed as JSON:
|
|
|
103
176
|
|
|
104
177
|
```json
|
|
105
178
|
{"log": true, "log_path": "/path/bindery_repair.log", "max_size_mb": 150,
|
|
106
|
-
"epubcheck_path": null}
|
|
179
|
+
"max_log_mb": 2, "epubcheck_path": null}
|
|
107
180
|
```
|
|
108
181
|
|
|
182
|
+
The log rotates at `max_log_mb` (default 2MB, one `.old` generation kept;
|
|
183
|
+
`0` disables rotation), so years of imports cannot grow it without bound.
|
|
184
|
+
|
|
109
185
|
`epubcheck_path` opts into the experimental on-PATH validation mode: the
|
|
110
186
|
repaired copy is re-measured with your epubcheck binary and refused unless it
|
|
111
187
|
is no worse than the original. It defaults to off; the default pass is safe
|
|
@@ -113,16 +189,17 @@ without a gate, and the mode costs two epubcheck runs per imported book.
|
|
|
113
189
|
|
|
114
190
|
## Usage
|
|
115
191
|
|
|
116
|
-
|
|
192
|
+
Five verbs: `bindery repair` fixes a single EPUB epubcheck-gated, `bindery audit`
|
|
117
193
|
reports content flaws without touching anything, `bindery library` sweeps a Calibre
|
|
118
194
|
library tree (dry run by default; `--apply` replaces accepted books atomically in place),
|
|
119
|
-
|
|
120
|
-
vetting, phase-3 scoped post-import repair) for a calling agent
|
|
195
|
+
`bindery run` composes the acquisition pathway's EPUB slices (phase-1 pre-import
|
|
196
|
+
vetting, phase-3 scoped post-import repair) for a calling agent, and `bindery doctor`
|
|
197
|
+
checks the installation (see Install).
|
|
121
198
|
The sections below take each in turn.
|
|
122
199
|
|
|
123
200
|
## Auditing
|
|
124
201
|
|
|
125
|
-
bindery-cli
|
|
202
|
+
bindery-cli's auditing tool inspects EPUB body text for non-schema flaws that epubcheck cannot catch. It extracts and analyzes the visible text to detect content issues, producing console reports that can be used to filter your library or feed into `bindery repair`.
|
|
126
203
|
|
|
127
204
|
Scan a Calibre library for specific issues:
|
|
128
205
|
|
|
@@ -156,7 +233,7 @@ Audits can also be run on a directory of loose `.epub` files by passing the path
|
|
|
156
233
|
bindery audit pagenumbers /path/to/loose/epubs
|
|
157
234
|
```
|
|
158
235
|
|
|
159
|
-
Or on one or more library books by Calibre id (comma-separated)
|
|
236
|
+
Or on one or more library books by Calibre id (comma-separated), fetched through cquarry's single-entity `get_book()` (no library-wide scan), with the EPUB resolved via cquarry's own path logic:
|
|
160
237
|
```sh
|
|
161
238
|
cd ~/docs/Calibre\ Library && bindery audit all --id 1234,1235
|
|
162
239
|
```
|
|
@@ -168,14 +245,16 @@ bindery audit all ~/Downloads/epubs --json vetting-report.json
|
|
|
168
245
|
```
|
|
169
246
|
With `--id`, `--json` accepts exactly one book id (each single-book run writes the file wholesale).
|
|
170
247
|
|
|
248
|
+
**Thresholds:** `--min-chars N` and `--thin-chars N` are the EMPTY and THIN body-text thresholds (defaults 2000 and 20000); `--min-chars` above `--thin-chars` is a usage error. `--max-doc-chars N` is the monolithic-document flag threshold (default 300,000). All three belong to the audit parser only (`bindery library --min-chars …` is an unknown-flag error).
|
|
249
|
+
|
|
171
250
|
**Spine-integrity reporting:** Both `audit` and `library` reports now classify manifest/NCX references that point to absent files. A `convention` verdict means the ToC is bloated but the present documents form a consecutive chapter span (e.g., the Wandering Inn official-build pattern, safe). A `fragment` verdict means the span itself is broken.
|
|
172
251
|
|
|
173
|
-
**Archive integrity:** every audit fully reads each archive entry (CRC + decompression), so a damaged download is reported CORRUPT
|
|
252
|
+
**Archive integrity:** every audit fully reads each archive entry (CRC + decompression), so a damaged download is reported CORRUPT, with the first broken entry named, instead of being mislabeled EMPTY by `emptytext`. `library --sweep` splits its `unreadable` bucket into `not_a_zip` / `truncated` / `encrypted` / `corrupt_entry`, so the right disease is visible without leaving the sweep.
|
|
174
253
|
|
|
175
254
|
### Tagging flagged books (opt-in)
|
|
176
255
|
|
|
177
256
|
By default an audit writes nothing. With `--tag TAG` (library mode only), every book the audit
|
|
178
|
-
flags is tagged in `metadata.db` through cquarry's trigger-safe write module
|
|
257
|
+
flags is tagged in `metadata.db` through cquarry's trigger-safe write module, useful for piping
|
|
179
258
|
flagged books back into Calibre views:
|
|
180
259
|
|
|
181
260
|
```sh
|
|
@@ -186,7 +265,7 @@ The audit itself stays read-only; only the final tagging pass writes, it skips b
|
|
|
186
265
|
carry the tag, and THIN emptytext advisories are never tagged. Close Calibre first so the write
|
|
187
266
|
does not fight its lock. Tagged books are recorded in Calibre's `metadata_dirtied` queue, so the
|
|
188
267
|
desktop app regenerates their sidecar `.opf`s (and re-pushes metadata to wireless readers) on its
|
|
189
|
-
next startup
|
|
268
|
+
next startup: no manual resync needed.
|
|
190
269
|
|
|
191
270
|
|
|
192
271
|
Repair one book to a new file (gated; writes only if it is an improvement):
|
|
@@ -195,8 +274,15 @@ Repair one book to a new file (gated; writes only if it is an improvement):
|
|
|
195
274
|
bindery repair broken.epub # -> "broken (repaired).epub"
|
|
196
275
|
bindery repair broken.epub fixed.epub
|
|
197
276
|
bindery repair scanned.epub --strip-pagination # also remove baked-in page numbers
|
|
277
|
+
bindery repair stubs.epub --strip-stub-docs # drop repeated placeholder chapters
|
|
278
|
+
bindery repair broken.epub fixed.epub --json report.json # machine-readable record
|
|
198
279
|
```
|
|
199
280
|
|
|
281
|
+
`repair --json FILE` writes one record in the `library --json` per-book
|
|
282
|
+
vocabulary (`status`, `applied`, `before`/`after` counts, the fix `summary`),
|
|
283
|
+
on every processing outcome including nochange and reject: a calling agent
|
|
284
|
+
gets the same facts the console lines carry.
|
|
285
|
+
|
|
200
286
|
Scan a Calibre library and see what would be fixed, writing nothing:
|
|
201
287
|
|
|
202
288
|
```sh
|
|
@@ -219,17 +305,16 @@ bindery library ~/docs/Calibre\ Library --only all --apply --all --install-to-ca
|
|
|
219
305
|
- `--id <ids>` limits the sweep to a comma-separated list of Calibre book IDs, skipping the full library walk.
|
|
220
306
|
- `--audit CSV` (the `fatals,errors,warnings,path` format produced by an epubcheck sweep) skips clean books so a run is fast. Paths are resolved on both sides, and a CSV that matches nothing triggers a loud warning instead of silently selecting zero books.
|
|
221
307
|
- `--limit N` processes at most N candidates and stops opening archives after them (lazy within one worker window). A value below 1 is a usage error.
|
|
222
|
-
- `--min-chars N` and `--thin-chars N` are the audit's EMPTY and THIN body-text thresholds (defaults 2000 and 20000); `--min-chars` above `--thin-chars` is a usage error. `--max-doc-chars N` is the audit monolithic-document flag threshold (default 300,000).
|
|
223
308
|
- `--sweep` replaces the CSV step entirely: it runs a live epubcheck sweep for candidate selection and reuses each result as that book's before-measurement, so no book is checked twice. bindery-cli may serve checks from a small persistent Java daemon (see the gate section above for how it is bounded and how it fails safe); wherever the daemon cannot serve, the run falls back to the standard per-book epubcheck subprocess at the usual seconds per book. Combine with `--only fatals` for a self-contained "find and fix the broken books" run.
|
|
224
|
-
- `--workers N` runs the sweep's candidate pass through N concurrent epubcheck workers (default 1: serial, unchanged). The subprocess releases the GIL, so threads parallelize the oracle
|
|
309
|
+
- `--workers N` runs the sweep's candidate pass through N concurrent epubcheck workers (default 1: serial, unchanged). The subprocess releases the GIL, so threads parallelize the oracle truthfully; books are checked in windows of N consumed in order, so the candidate set matches the serial sweep and `--limit` stays lazy within one window of overshoot. The repair phase stays serial: that is where the shared workdir and the atomic-replacement contract live.
|
|
225
310
|
- `--json FILE` writes a machine-readable report of the whole run (per-book status, before/after counts, applied flag, summary totals). `--manual-list FILE` writes the paths of every book that was not auto-repaired, one per line, ready for manual follow-up.
|
|
226
|
-
- `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file.
|
|
311
|
+
- `--apply` is required to write; the default is a dry run. `--backup DIR` mirrors originals before replacing; `--backup-inplace` writes `.epub.bak` beside each file. `--backup-keep N` bounds the rotation: at most N backup files per book (the author original `.epub.bak` is never deleted), so a years-long apply habit cannot grow backups without bound.
|
|
227
312
|
- `--install-to-calibre` resolves the Calibre book id from `metadata.db` via cquarry (one read-only path→id map per run), so a hand-renamed `Author/Title (id)/` directory can never send the repaired file to the wrong book. The directory-name regex is only a no-catalog fallback; with neither, the file is saved atomically in place.
|
|
228
313
|
- `--install-to-calibre` installs the repaired EPUB through cquarry's write module: the file is placed atomically at the catalogued path (same filename, Calibre's layout never changes) and the format row is re-registered in one transaction, so the stored size stays truthful and the book lands in `metadata_dirtied`; Calibre regenerates its sidecar .opf on next start. No external `calibredb` call is involved. It falls back to atomic file replacement if the Calibre database ID cannot be resolved, and a database failure degrades to the in-place save with a warning. A directory-name-guessed id is trusted only when `metadata.db` corroborates it (the book exists and the file is that book's catalogued EPUB); an uncatalogued stray file inside a book directory is saved in place and the catalog row is left untouched.
|
|
229
314
|
- `--all` automatically turns on all opt-in non-fatal fixes and lossy strips (pagination, watermarks, bad attributes, unknown entities, image alt tags, etc.) in a single run.
|
|
230
315
|
- Only the `.epub` is replaced. `metadata.opf`, `cover.jpg`, and `metadata.db` are left for Calibre's Quality Check sync to reconcile.
|
|
231
316
|
- A per-book progress line goes to stderr (stdout stays a clean report); `--quiet` suppresses it. A corrupt or unreadable book is reported and skipped, never aborting the sweep.
|
|
232
|
-
- Exit codes
|
|
317
|
+
- Exit codes for `repair`, `library`, and `run`: 0 for a clean sweep, 1 for a usage error, 2 when any book was rejected, unreadable, or failed epubcheck (for scripts and cron). Note that argparse-level misuse (an unknown flag or a malformed argument) exits with 2 before any validation runs; the tool's own usage validations exit 1. The `audit` verb runs its own contract, the inverse shape: 0 clean, 1 when any book was flagged (a flag is the audit's trouble), 2 for its own usage errors (a path that is not a directory, a directory with no EPUBs, an unknown book id).
|
|
233
318
|
- A `partial` book (improved but still unable to open) is reported for manual follow-up and fails the run with exit 2, in `library`, `run phase1`, and `run phase3` alike: a book that still needs a human is trouble.
|
|
234
319
|
- `repair` refuses to overwrite an existing output file unless `--force` is given.
|
|
235
320
|
|
|
@@ -272,7 +357,7 @@ cd ~/docs/Calibre\ Library && bindery run phase3 --ids 5071,5072 --json phase3-r
|
|
|
272
357
|
|
|
273
358
|
`scripts/` holds standalone, read-only utilities that are useful for EPUB maintenance but fall outside bindery-cli's repair contract (fixing what they find would be a content change, which bindery-cli makes only via the opt-in `--strip-pagination`):
|
|
274
359
|
|
|
275
|
-
- `fast_sweep.py`: compiles and drives `FastSweep.java`, a parallel JVM epubcheck sweep that pays JVM startup once instead of per book (a 7,000-book dry-run sweep drops from hours to minutes). `--mode=audit` emits the `fatals,errors,warnings,path` CSV that `bindery library --audit` reads; `--mode=extract` emits per-book error codes for aggregation (`--summary` prints the per-code report). The epubcheck jar is located from the `epubcheck` launcher or `EPUBCHECK_JAR`; the harness compiles once (`--release 25`) and is cached on mtime.
|
|
360
|
+
- `fast_sweep.py`: compiles and drives `FastSweep.java`, a parallel JVM epubcheck sweep that pays JVM startup once instead of per book (a 7,000-book dry-run sweep drops from hours to minutes). `--mode=audit` emits the `fatals,errors,warnings,path` CSV that `bindery library --audit` reads, with the counts taken from the same aggregated JSON surface the gate measures (not the raw occurrence counters), so selection and gating always agree; `--mode=extract` emits per-book error codes for aggregation (`--summary` prints the per-code report). The epubcheck jar is located from the `epubcheck` launcher or `EPUBCHECK_JAR`; the harness compiles once (`--release 25`) and is cached on mtime.
|
|
276
361
|
|
|
277
362
|
The older `find_*.py` detection wedges and the `sweep.sh`/`FastSweepExtract.java` prototypes were removed: every detection they performed is now a shipped, tested flag (missing images are `--prune-missing-resources`, and the rest of the RSC-005/PKG-010 family shipped through Phases 6-12), and keeping two copies meant future behavior changes landed in only one of them. Git history preserves them.
|
|
278
363
|
|