readalign 1.0.0__tar.gz → 1.2.0__tar.gz

{readalign-1.0.0 → readalign-1.2.0}/.gitignore

@@ -1,20 +1,20 @@
-__pycache__/
-*.py[cod]
-*.egg-info/
-.eggs/
-build/
-dist/
-.venv/
-venv/
-.pytest_cache/
-.ruff_cache/
-.coverage
-htmlcov/
-*.epub
-!examples/aesop/*.epub
-readalign-report.json
-*-sidecars/
-.readalign-cache/
-examples/output/
-.DS_Store
-Thumbs.db
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.venv/
+venv/
+.pytest_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+*.epub
+!examples/aesop/*.epub
+readalign-report.json
+*-sidecars/
+.readalign-cache/
+examples/output/
+.DS_Store
+Thumbs.db

readalign-1.2.0/CHANGELOG.md

@@ -0,0 +1,94 @@
+# Changelog
+
+## 1.2.0 - 2026-10-22
+
+- New: `--report PATH` selects a separate JSON report for each batch build, with guards
+  against overwriting input audio, the input/output EPUB or generated sidecars.
+- Fixed: colliding overlay names and URL-special characters in EPUB resource paths.
+  Generated audio and CSS also avoid overwriting unrelated existing resources.
+- Fixed: transcript cache identity now hashes prepared audio bytes and includes chapter
+  boundaries and the effective language. Malformed cached words are ignored instead of crashing.
+- Fixed: EPUB, report and cache writes replace their destinations only after a complete write.
+- Fixed: work-directory audio aliases, invalid durations, chapter marks and beam sizes are
+  rejected; missing epubcheck paths fail before transcription.
+- Fixed: `check` rejects unmanifested audio, overlays pointing outside their associated document,
+  empty overlays, duplicate manifest ids and mismatched durations. Corrupt archives produce
+  readable errors; audio-probe scratch filenames never come from archive member names.
+- New: `--min-token-match FRACTION` optionally gates builds on the fraction of book tokens
+  matched to narration. Values range from 0 to 1; the default 0 preserves existing behaviour.
+  A failed gate exits 5 after writing the EPUB, report and enabled sidecars. Its threshold and
+  verdict are recorded under `token_match_gate`; drift and epubcheck failures retain precedence.
+- Fixed: chapters with identical basenames in different EPUB directories could overwrite each
+  other's WebVTT sidecars. Colliding names now gain a numeric suffix, including case-only
+  collisions on Windows, without changing filenames for non-colliding exports.
+- Fixed: the ebook overwrite guard now recognises symbolic links, hard links and path aliases.
+- Fixed: non-finite drift thresholds such as NaN and infinity are rejected before processing.
+
+## 1.1.0 - 2026-10-04
+
+- New: `--verify [JAR]` runs epubcheck on the result right after the build, prints its verdict,
+  records it under `verify` in the report, and exits 4 when it finds errors. epubcheck needs
+  Java, so the flag is opt-in: with a path it uses that jar, without one it searches `PATH`,
+  `EPUBCHECK_HOME` and `JAVA_HOME`.
+- New: every unaligned region in the report carries an `excerpt` quoting the book's own words,
+  so a stretch that never matched the narration is recognisable without a token diff.
+- Fixed: `--audio` accepted a glob in the documentation but not in the code, so a pattern like
+  `book-*.mp3` failed with "audio not found" on Windows (and anywhere the shell was not asked
+  to expand it). Patterns are now expanded by readalign itself, in natural order, with the same
+  DRM refusal as a directory.
+- Fixed: `readalign check` raised a raw traceback on input that was a valid zip but broken XML:
+  an unparseable `META-INF/container.xml` or package document is now one line and exit 1, and
+  an unparseable overlay or content document is reported as a finding like any other fault
+  instead of crashing the walk.
+- Fixed: rebuilding over an EPUB that already carried overlays kept the previous run's audio
+  manifest item, so a re-run whose prepared audio changed name or format shipped the old file
+  as dead weight (a full extra audiobook when the suffix drifted). A re-run now drops the
+  previous audio and reuses the same manifest ids, the way it already did for overlays and the
+  stylesheet.
+- Fixed: manifest items added while writing an output were not registered in the package's
+  manifest index, so two builds over one package could hand out the same item id twice.
+- Fixed: the transcript cache key ignored `--beam-size` (and the window plan), so changing the
+  beam silently served a transcript made with the old setting. Keys now cover every setting
+  that changes the transcription, which invalidates caches written by 1.0.0 once.
+- Fixed: a sentence ending in the word "no" never split from the next one ("He said no. She
+  left."), because "no" sat in the abbreviation table. Numbered labels like "No. 5" still stay
+  together: the digit after the period is not an uppercase word.
+- Fixed: the warning for a failed cache write bypassed the `--quiet` log and printed straight
+  to stderr; it now goes through the log like every other progress line.
+- DRM refusals name the file they refused, which matters when a directory of audio contains
+  one protected straggler.
+- `readalign check` says "and N more" after its first twenty warnings, as it already did for
+  errors, instead of truncating silently.
+
+## 1.0.0 - 2026-09-18
+
+First release.
+
+- `readalign build` turns a DRM-free audiobook and the matching DRM-free EPUB into one EPUB 3
+  with conformant Media Overlays: SMIL per content document, `media-overlay` on the manifest
+  items, `media:duration` per overlay plus a total, active-class metadata and a highlight
+  stylesheet. The overlay's `seq` elements mirror the document's nesting, so both skippability
+  (footnotes, endnotes, page numbers, sidebars, annotations) and escapability (tables, lists,
+  figures) work. The archive is written mimetype-first and uncompressed as OCF requires.
+- `readalign check` validates the overlays in any EPUB: fragment ids resolve, audio is
+  manifested, no `clipEnd` runs past the duration ffprobe reports, declared durations match the
+  clips. `--json` for scripting.
+- Transcription with faster-whisper, CUDA when available and CPU otherwise, word timestamps,
+  and a per-file transcript cache keyed by content and settings. A GPU that reports itself
+  present but cannot load its CUDA libraries falls back to the CPU with one warning line
+  naming what to install, rather than a traceback.
+- Anchored alignment: unique shared n-grams, longest increasing subsequence, diff only in the
+  gaps. No global dynamic-programming matrix, and audio is streamed rather than buffered on the
+  way both in and out, so a 7.5 hour book builds in about 130 MB of memory and opening the
+  205 MB result costs 33 MB. Two runs of the same build produce byte-identical archives.
+- Sidecars: one WebVTT per chapter and audio file, plus `align.json`. `readalign-report.json`
+  records coverage, drift, per-document statistics and every region that did not match.
+- Handles one file spanning several chapters, several files in one chapter, audio chapter counts
+  that disagree with the spine, narrator front and back matter with no text counterpart, notes
+  and page-break markers, existing ids, non-UTF8 XHTML, and clip ends past the real duration.
+- DRM-protected input (`.aax`, `.aa`, `.acsm`, an encrypted EPUB) exits 2 and is never opened.
+- Wrong input gets one line and an exit code, never a traceback: a missing file, an unzipped
+  book passed as a directory, something that is not a zip, an EPUB 2 package, an empty audio
+  directory, or a file ffprobe cannot read.
+- Optional `readalign[refine]` extra adds a CTC forced-alignment pass behind `--refine`. The
+  MMS-300m aligner it uses is CC BY-NC 4.0, so it is never downloaded by the default path.

{readalign-1.0.0 → readalign-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: readalign
-Version: 1.0.0
+Version: 1.2.0
 Summary: Turn an audiobook and the matching ebook you own into one EPUB 3 with Media Overlays.
 Project-URL: Homepage, https://github.com/Booyaka101/readalign
 Project-URL: Repository, https://github.com/Booyaka101/readalign
@@ -50,7 +50,8 @@ readalign build --audio audiobook/ --epub book.epub --out read-along.epub
   `media-overlay` links, `media:duration` metadata and a highlight stylesheet. It passes
   epubcheck 5.2.1 with zero messages.
 - **`readalign-report.json`** next to the output: coverage, drift, per-document statistics, which
-  documents got no overlay and which regions never matched.
+  documents got no overlay and which regions never matched, each quoted with a short excerpt of
+  the book's own words so a bad match is obvious without opening the JSON.
 - **sidecars** (`<name>-sidecars/`): one WebVTT file per chapter and audio file, plus `align.json`
   with every sentence's id, clip times, confidence and text. A rerun clears the `.vtt` files and
   `align.json` it wrote last time, so nothing stale is left behind. Turn them off with
@@ -93,7 +94,7 @@ Two minutes of audio is not worth spinning up a GPU for, hence `--device cpu`. R
 from that command, on a cold cache:
 
 ```
-readalign 1.0.0
+readalign 1.1.0
 reading examples/aesop/aesop-two-fables.epub
   2 content documents, 18 sentences
 preparing 1 audio file(s)
@@ -150,6 +151,12 @@ It is worth running the official validator too:
 java -jar epubcheck.jar aligned.epub
 ```
 
+Or let readalign do it: `--verify` runs epubcheck on the result right after writing it, prints
+its verdict, records it in the report, and exits 4 if it found errors. It needs Java and the
+epubcheck jar, which is why it is opt-in: pass the jar's path (`--verify C:/tools/epubcheck.jar`)
+or leave the value off and readalign will look for one on `PATH`, in `EPUBCHECK_HOME` and in
+`JAVA_HOME`.
+
 ## A full-length run
 
 The 25-file LibriVox recording of *Frankenstein* (7 h 26 m) against the Standard Ebooks edition,
@@ -203,13 +210,23 @@ Drift is how far a sentence's start could be wrong, in seconds.
 `--drift-threshold` (default 2.5s) is a gate on the maximum, not the average. The EPUB is always
 written; exceeding the threshold only changes the exit code to 3 so a script can catch it.
 
+For batch builds, `--min-token-match 0.9` also requires at least 90% of the ebook's normalised
+tokens to match the narration. This optional gate defaults to 0 (disabled). A failed gate exits 5
+but keeps the EPUB, report and enabled sidecars for inspection. The report's `token_match_gate`
+records the threshold and verdict using the unrounded match fraction. Unnarrated front matter
+and abridgements lower this score, so choose a threshold appropriate to your books. This is a
+screening aid, not proof that every sentence is correctly timed. Drift and epubcheck failures
+take precedence over this gate when more than one check fails.
+
 ## How it works
 
 1. **ffprobe** reads every audio file. FLAC, WAV, ALAC and WMA are transcoded to AAC because
    EPUB 3 only takes MP3, MP4 audio and Opus; MP3, M4A/M4B and Opus are stream-copied, so no
    audio is re-encoded unless it has to be.
 2. **faster-whisper** transcribes with word timestamps. Transcripts are cached per file, keyed by
-   content and settings, so a second run over the same audiobook costs nothing.
+   a streamed SHA-256 of the prepared audio, chapter boundaries and recogniser settings.
+   A second run avoids transcription, but still prepares and hashes the audio. Older cache keys
+   are invalidated once; malformed entries are ignored and rebuilt.
 3. Both sides are normalised to comparable tokens: case folded, accents stripped, punctuation
    dropped, digits and roman numerals spelled out ("Chapter XVIII" and "chapter eighteen" become
    the same tokens).
@@ -233,9 +250,11 @@ written; exceeding the threshold only changes the exit code to 3 so a script can
 --audio PATH             a file, a directory of numbered files, or a glob
 --epub PATH              the matching DRM-free EPUB
 --out PATH               the EPUB to write
+--report PATH            JSON report path (default: readalign-report.json beside --out)
 --model NAME             faster-whisper model (default: large-v3-turbo)
 --language CODE          ISO code of the narration (default: detect)
 --drift-threshold SECS   exit 3 if any sentence drifts more than this (default: 2.5)
+--min-token-match N      exit 5 if matched book-token fraction is below N (0 to 1; default: 0)
 --sidecars/--no-sidecars write WebVTT and align.json next to the output (default: on)
 --dry-run                probe the inputs and print the plan, transcribe nothing
 --narrator NAME          value for the media:narrator metadata
@@ -244,6 +263,9 @@ written; exceeding the threshold only changes the exit code to 3 so a script can
 --beam-size N            whisper beam size (default: 5)
 --no-vad                 disable the voice-activity filter
 --audio-bitrate RATE     bitrate when audio has to be transcoded (default: 96k)
+--verify [JAR]           validate the result with epubcheck after writing it (needs Java);
+                         pass its jar path, or leave the value off to search PATH,
+                         EPUBCHECK_HOME and JAVA_HOME for one
 --work-dir PATH          scratch directory instead of a temporary one
 --cache-dir PATH         transcript cache location
 --no-cache               neither read nor write the transcript cache
@@ -300,6 +322,8 @@ against the reference Media Overlays title from the EPUB test suite at
 | 1 | bad input, missing file, missing ffmpeg, nothing could be aligned |
 | 2 | the input is DRM-protected |
 | 3 | the EPUB was written but max drift exceeded `--drift-threshold` |
+| 4 | the EPUB was written but epubcheck reported problems (only with `--verify`) |
+| 5 | the EPUB was written but token match was below `--min-token-match` |
 | 130 | interrupted |
 
 Failures print one line, not a traceback.

{readalign-1.0.0 → readalign-1.2.0}/README.md

@@ -17,7 +17,8 @@ readalign build --audio audiobook/ --epub book.epub --out read-along.epub
   `media-overlay` links, `media:duration` metadata and a highlight stylesheet. It passes
   epubcheck 5.2.1 with zero messages.
 - **`readalign-report.json`** next to the output: coverage, drift, per-document statistics, which
-  documents got no overlay and which regions never matched.
+  documents got no overlay and which regions never matched, each quoted with a short excerpt of
+  the book's own words so a bad match is obvious without opening the JSON.
 - **sidecars** (`<name>-sidecars/`): one WebVTT file per chapter and audio file, plus `align.json`
   with every sentence's id, clip times, confidence and text. A rerun clears the `.vtt` files and
   `align.json` it wrote last time, so nothing stale is left behind. Turn them off with
@@ -60,7 +61,7 @@ Two minutes of audio is not worth spinning up a GPU for, hence `--device cpu`. R
 from that command, on a cold cache:
 
 ```
-readalign 1.0.0
+readalign 1.1.0
 reading examples/aesop/aesop-two-fables.epub
   2 content documents, 18 sentences
 preparing 1 audio file(s)
@@ -117,6 +118,12 @@ It is worth running the official validator too:
 java -jar epubcheck.jar aligned.epub
 ```
 
+Or let readalign do it: `--verify` runs epubcheck on the result right after writing it, prints
+its verdict, records it in the report, and exits 4 if it found errors. It needs Java and the
+epubcheck jar, which is why it is opt-in: pass the jar's path (`--verify C:/tools/epubcheck.jar`)
+or leave the value off and readalign will look for one on `PATH`, in `EPUBCHECK_HOME` and in
+`JAVA_HOME`.
+
 ## A full-length run
 
 The 25-file LibriVox recording of *Frankenstein* (7 h 26 m) against the Standard Ebooks edition,
@@ -170,13 +177,23 @@ Drift is how far a sentence's start could be wrong, in seconds.
 `--drift-threshold` (default 2.5s) is a gate on the maximum, not the average. The EPUB is always
 written; exceeding the threshold only changes the exit code to 3 so a script can catch it.
 
+For batch builds, `--min-token-match 0.9` also requires at least 90% of the ebook's normalised
+tokens to match the narration. This optional gate defaults to 0 (disabled). A failed gate exits 5
+but keeps the EPUB, report and enabled sidecars for inspection. The report's `token_match_gate`
+records the threshold and verdict using the unrounded match fraction. Unnarrated front matter
+and abridgements lower this score, so choose a threshold appropriate to your books. This is a
+screening aid, not proof that every sentence is correctly timed. Drift and epubcheck failures
+take precedence over this gate when more than one check fails.
+
 ## How it works
 
 1. **ffprobe** reads every audio file. FLAC, WAV, ALAC and WMA are transcoded to AAC because
    EPUB 3 only takes MP3, MP4 audio and Opus; MP3, M4A/M4B and Opus are stream-copied, so no
    audio is re-encoded unless it has to be.
 2. **faster-whisper** transcribes with word timestamps. Transcripts are cached per file, keyed by
-   content and settings, so a second run over the same audiobook costs nothing.
+   a streamed SHA-256 of the prepared audio, chapter boundaries and recogniser settings.
+   A second run avoids transcription, but still prepares and hashes the audio. Older cache keys
+   are invalidated once; malformed entries are ignored and rebuilt.
 3. Both sides are normalised to comparable tokens: case folded, accents stripped, punctuation
    dropped, digits and roman numerals spelled out ("Chapter XVIII" and "chapter eighteen" become
    the same tokens).
@@ -200,9 +217,11 @@ written; exceeding the threshold only changes the exit code to 3 so a script can
 --audio PATH             a file, a directory of numbered files, or a glob
 --epub PATH              the matching DRM-free EPUB
 --out PATH               the EPUB to write
+--report PATH            JSON report path (default: readalign-report.json beside --out)
 --model NAME             faster-whisper model (default: large-v3-turbo)
 --language CODE          ISO code of the narration (default: detect)
 --drift-threshold SECS   exit 3 if any sentence drifts more than this (default: 2.5)
+--min-token-match N      exit 5 if matched book-token fraction is below N (0 to 1; default: 0)
 --sidecars/--no-sidecars write WebVTT and align.json next to the output (default: on)
 --dry-run                probe the inputs and print the plan, transcribe nothing
 --narrator NAME          value for the media:narrator metadata
@@ -211,6 +230,9 @@ written; exceeding the threshold only changes the exit code to 3 so a script can
 --beam-size N            whisper beam size (default: 5)
 --no-vad                 disable the voice-activity filter
 --audio-bitrate RATE     bitrate when audio has to be transcoded (default: 96k)
+--verify [JAR]           validate the result with epubcheck after writing it (needs Java);
+                         pass its jar path, or leave the value off to search PATH,
+                         EPUBCHECK_HOME and JAVA_HOME for one
 --work-dir PATH          scratch directory instead of a temporary one
 --cache-dir PATH         transcript cache location
 --no-cache               neither read nor write the transcript cache
@@ -267,6 +289,8 @@ against the reference Media Overlays title from the EPUB test suite at
 | 1 | bad input, missing file, missing ffmpeg, nothing could be aligned |
 | 2 | the input is DRM-protected |
 | 3 | the EPUB was written but max drift exceeded `--drift-threshold` |
+| 4 | the EPUB was written but epubcheck reported problems (only with `--verify`) |
+| 5 | the EPUB was written but token match was below `--min-token-match` |
 | 130 | interrupted |
 
 Failures print one line, not a traceback.

{readalign-1.0.0 → readalign-1.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "readalign"
-version = "1.0.0"
+version = "1.2.0"
 description = "Turn an audiobook and the matching ebook you own into one EPUB 3 with Media Overlays."
 readme = "README.md"
 requires-python = ">=3.11"

{readalign-1.0.0 → readalign-1.2.0}/src/readalign/__init__.py

@@ -1,3 +1,3 @@
 """readalign: build an EPUB 3 with Media Overlays from an audiobook and the matching ebook."""
 
-__version__ = "1.0.0"
+__version__ = "1.2.0"

{readalign-1.0.0 → readalign-1.2.0}/src/readalign/align.py

@@ -175,6 +175,14 @@ def _track_for(tracks: list[AudioTrack], moment: float) -> AudioTrack:
     return tracks[-1]
 
 
+def _excerpt(raw: list[str], start: int, end: int, limit: int = 12) -> str:
+    """A short human-readable preview of one unaligned stretch, for the report."""
+    words = raw[max(0, start): max(0, start, end)]
+    if len(words) <= limit:
+        return " ".join(words)
+    return " ".join(words[:limit]) + " ..."
+
+
 def align(
     sentences: list[Sentence],
     words: list[Word],
@@ -192,11 +200,14 @@ def align(
         )
 
     book_tokens: list[str] = []
+    book_raw: list[str] = []
     sentence_spans: list[tuple[int, int]] = []
     for sentence in sentences:
-        tokens, _ = tokenize_words(sentence.words)
+        tokens, origins = tokenize_words(sentence.words)
         sentence_spans.append((len(book_tokens), len(book_tokens) + len(tokens)))
         book_tokens.extend(tokens)
+        # The raw word behind every token, so the report can quote the book's own spelling.
+        book_raw.extend(sentence.words[origin] for origin in origins)
 
     asr_tokens, asr_origin = tokenize_words([word.text for word in words])
     if not book_tokens:
@@ -224,6 +235,7 @@ def align(
         unaligned_regions=[
             {
                 "book_tokens": [region[0], region[1]],
+                "excerpt": _excerpt(book_raw, region[0], region[1]),
                 "audio_seconds": [
                     round(words[asr_origin[min(region[2], len(asr_origin) - 1)]].start, 2),
                     round(words[asr_origin[min(region[3] - 1, len(asr_origin) - 1)]].end, 2),