readalign 1.1.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {readalign-1.1.0 → readalign-1.2.0}/.gitignore +20 -20
- {readalign-1.1.0 → readalign-1.2.0}/CHANGELOG.md +24 -0
- {readalign-1.1.0 → readalign-1.2.0}/PKG-INFO +15 -2
- {readalign-1.1.0 → readalign-1.2.0}/README.md +14 -1
- {readalign-1.1.0 → readalign-1.2.0}/pyproject.toml +1 -1
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/__init__.py +1 -1
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/asr.py +32 -13
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/audio.py +9 -3
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/cli.py +105 -24
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/clock.py +15 -9
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/epub.py +674 -671
- readalign-1.2.0/src/readalign/fs.py +64 -0
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/members.py +72 -72
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/package.py +73 -54
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/sidecar.py +10 -5
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/smil.py +4 -3
- {readalign-1.1.0 → readalign-1.2.0}/tests/test_asr.py +51 -0
- {readalign-1.1.0 → readalign-1.2.0}/tests/test_audio.py +38 -1
- {readalign-1.1.0 → readalign-1.2.0}/tests/test_cli.py +114 -0
- readalign-1.2.0/tests/test_fs.py +51 -0
- {readalign-1.1.0 → readalign-1.2.0}/tests/test_output.py +110 -0
- {readalign-1.1.0 → readalign-1.2.0}/tests/test_sidecar_report.py +33 -0
- readalign-1.1.0/src/readalign/fs.py +0 -22
- {readalign-1.1.0 → readalign-1.2.0}/LICENSE +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/scripts/fetch_samples.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/__main__.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/align.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/errors.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/refine.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/report.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/sentences.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/textnorm.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/src/readalign/verify.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/tests/conftest.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/tests/test_align.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/tests/test_epub.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/tests/test_text.py +0 -0
- {readalign-1.1.0 → readalign-1.2.0}/tests/test_verify.py +0 -0
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
__pycache__/
|
|
2
|
-
*.py[cod]
|
|
3
|
-
*.egg-info/
|
|
4
|
-
.eggs/
|
|
5
|
-
build/
|
|
6
|
-
dist/
|
|
7
|
-
.venv/
|
|
8
|
-
venv/
|
|
9
|
-
.pytest_cache/
|
|
10
|
-
.ruff_cache/
|
|
11
|
-
.coverage
|
|
12
|
-
htmlcov/
|
|
13
|
-
*.epub
|
|
14
|
-
!examples/aesop/*.epub
|
|
15
|
-
readalign-report.json
|
|
16
|
-
*-sidecars/
|
|
17
|
-
.readalign-cache/
|
|
18
|
-
examples/output/
|
|
19
|
-
.DS_Store
|
|
20
|
-
Thumbs.db
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
*.egg-info/
|
|
4
|
+
.eggs/
|
|
5
|
+
build/
|
|
6
|
+
dist/
|
|
7
|
+
.venv/
|
|
8
|
+
venv/
|
|
9
|
+
.pytest_cache/
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
.coverage
|
|
12
|
+
htmlcov/
|
|
13
|
+
*.epub
|
|
14
|
+
!examples/aesop/*.epub
|
|
15
|
+
readalign-report.json
|
|
16
|
+
*-sidecars/
|
|
17
|
+
.readalign-cache/
|
|
18
|
+
examples/output/
|
|
19
|
+
.DS_Store
|
|
20
|
+
Thumbs.db
|
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.2.0 - 2026-10-22
|
|
4
|
+
|
|
5
|
+
- New: `--report PATH` selects a separate JSON report for each batch build, with guards
|
|
6
|
+
against overwriting input audio, the input/output EPUB or generated sidecars.
|
|
7
|
+
- Fixed: colliding overlay names and URL-special characters in EPUB resource paths.
|
|
8
|
+
Generated audio and CSS also avoid overwriting unrelated existing resources.
|
|
9
|
+
- Fixed: transcript cache identity now hashes prepared audio bytes and includes chapter
|
|
10
|
+
boundaries and the effective language. Malformed cached words are ignored instead of crashing.
|
|
11
|
+
- Fixed: EPUB, report and cache writes replace their destinations only after a complete write.
|
|
12
|
+
- Fixed: work-directory audio aliases, invalid durations, chapter marks and beam sizes are
|
|
13
|
+
rejected; missing epubcheck paths fail before transcription.
|
|
14
|
+
- Fixed: `check` rejects unmanifested audio, overlays pointing outside their associated document,
|
|
15
|
+
empty overlays, duplicate manifest ids and mismatched durations. Corrupt archives produce
|
|
16
|
+
readable errors; audio-probe scratch filenames never come from archive member names.
|
|
17
|
+
- New: `--min-token-match FRACTION` optionally gates builds on the fraction of book tokens
|
|
18
|
+
matched to narration. Values range from 0 to 1; the default 0 preserves existing behaviour.
|
|
19
|
+
A failed gate exits 5 after writing the EPUB, report and enabled sidecars. Its threshold and
|
|
20
|
+
verdict are recorded under `token_match_gate`; drift and epubcheck failures retain precedence.
|
|
21
|
+
- Fixed: chapters with identical basenames in different EPUB directories could overwrite each
|
|
22
|
+
other's WebVTT sidecars. Colliding names now gain a numeric suffix, including case-only
|
|
23
|
+
collisions on Windows, without changing filenames for non-colliding exports.
|
|
24
|
+
- Fixed: the ebook overwrite guard now recognises symbolic links, hard links and path aliases.
|
|
25
|
+
- Fixed: non-finite drift thresholds such as NaN and infinity are rejected before processing.
|
|
26
|
+
|
|
3
27
|
## 1.1.0 - 2026-10-04
|
|
4
28
|
|
|
5
29
|
- New: `--verify [JAR]` runs epubcheck on the result right after the build, prints its verdict,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: readalign
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Turn an audiobook and the matching ebook you own into one EPUB 3 with Media Overlays.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Booyaka101/readalign
|
|
6
6
|
Project-URL: Repository, https://github.com/Booyaka101/readalign
|
|
@@ -210,13 +210,23 @@ Drift is how far a sentence's start could be wrong, in seconds.
|
|
|
210
210
|
`--drift-threshold` (default 2.5s) is a gate on the maximum, not the average. The EPUB is always
|
|
211
211
|
written; exceeding the threshold only changes the exit code to 3 so a script can catch it.
|
|
212
212
|
|
|
213
|
+
For batch builds, `--min-token-match 0.9` also requires at least 90% of the ebook's normalised
|
|
214
|
+
tokens to match the narration. This optional gate defaults to 0 (disabled). A failed gate exits 5
|
|
215
|
+
but keeps the EPUB, report and enabled sidecars for inspection. The report's `token_match_gate`
|
|
216
|
+
records the threshold and verdict using the unrounded match fraction. Unnarrated front matter
|
|
217
|
+
and abridgements lower this score, so choose a threshold appropriate to your books. This is a
|
|
218
|
+
screening aid, not proof that every sentence is correctly timed. Drift and epubcheck failures
|
|
219
|
+
take precedence over this gate when more than one check fails.
|
|
220
|
+
|
|
213
221
|
## How it works
|
|
214
222
|
|
|
215
223
|
1. **ffprobe** reads every audio file. FLAC, WAV, ALAC and WMA are transcoded to AAC because
|
|
216
224
|
EPUB 3 only takes MP3, MP4 audio and Opus; MP3, M4A/M4B and Opus are stream-copied, so no
|
|
217
225
|
audio is re-encoded unless it has to be.
|
|
218
226
|
2. **faster-whisper** transcribes with word timestamps. Transcripts are cached per file, keyed by
|
|
219
|
-
|
|
227
|
+
a streamed SHA-256 of the prepared audio, chapter boundaries and recogniser settings.
|
|
228
|
+
A second run avoids transcription, but still prepares and hashes the audio. Older cache keys
|
|
229
|
+
are invalidated once; malformed entries are ignored and rebuilt.
|
|
220
230
|
3. Both sides are normalised to comparable tokens: case folded, accents stripped, punctuation
|
|
221
231
|
dropped, digits and roman numerals spelled out ("Chapter XVIII" and "chapter eighteen" become
|
|
222
232
|
the same tokens).
|
|
@@ -240,9 +250,11 @@ written; exceeding the threshold only changes the exit code to 3 so a script can
|
|
|
240
250
|
--audio PATH a file, a directory of numbered files, or a glob
|
|
241
251
|
--epub PATH the matching DRM-free EPUB
|
|
242
252
|
--out PATH the EPUB to write
|
|
253
|
+
--report PATH JSON report path (default: readalign-report.json beside --out)
|
|
243
254
|
--model NAME faster-whisper model (default: large-v3-turbo)
|
|
244
255
|
--language CODE ISO code of the narration (default: detect)
|
|
245
256
|
--drift-threshold SECS exit 3 if any sentence drifts more than this (default: 2.5)
|
|
257
|
+
--min-token-match N exit 5 if matched book-token fraction is below N (0 to 1; default: 0)
|
|
246
258
|
--sidecars/--no-sidecars write WebVTT and align.json next to the output (default: on)
|
|
247
259
|
--dry-run probe the inputs and print the plan, transcribe nothing
|
|
248
260
|
--narrator NAME value for the media:narrator metadata
|
|
@@ -311,6 +323,7 @@ against the reference Media Overlays title from the EPUB test suite at
|
|
|
311
323
|
| 2 | the input is DRM-protected |
|
|
312
324
|
| 3 | the EPUB was written but max drift exceeded `--drift-threshold` |
|
|
313
325
|
| 4 | the EPUB was written but epubcheck reported problems (only with `--verify`) |
|
|
326
|
+
| 5 | the EPUB was written but token match was below `--min-token-match` |
|
|
314
327
|
| 130 | interrupted |
|
|
315
328
|
|
|
316
329
|
Failures print one line, not a traceback.
|
|
@@ -177,13 +177,23 @@ Drift is how far a sentence's start could be wrong, in seconds.
|
|
|
177
177
|
`--drift-threshold` (default 2.5s) is a gate on the maximum, not the average. The EPUB is always
|
|
178
178
|
written; exceeding the threshold only changes the exit code to 3 so a script can catch it.
|
|
179
179
|
|
|
180
|
+
For batch builds, `--min-token-match 0.9` also requires at least 90% of the ebook's normalised
|
|
181
|
+
tokens to match the narration. This optional gate defaults to 0 (disabled). A failed gate exits 5
|
|
182
|
+
but keeps the EPUB, report and enabled sidecars for inspection. The report's `token_match_gate`
|
|
183
|
+
records the threshold and verdict using the unrounded match fraction. Unnarrated front matter
|
|
184
|
+
and abridgements lower this score, so choose a threshold appropriate to your books. This is a
|
|
185
|
+
screening aid, not proof that every sentence is correctly timed. Drift and epubcheck failures
|
|
186
|
+
take precedence over this gate when more than one check fails.
|
|
187
|
+
|
|
180
188
|
## How it works
|
|
181
189
|
|
|
182
190
|
1. **ffprobe** reads every audio file. FLAC, WAV, ALAC and WMA are transcoded to AAC because
|
|
183
191
|
EPUB 3 only takes MP3, MP4 audio and Opus; MP3, M4A/M4B and Opus are stream-copied, so no
|
|
184
192
|
audio is re-encoded unless it has to be.
|
|
185
193
|
2. **faster-whisper** transcribes with word timestamps. Transcripts are cached per file, keyed by
|
|
186
|
-
|
|
194
|
+
a streamed SHA-256 of the prepared audio, chapter boundaries and recogniser settings.
|
|
195
|
+
A second run avoids transcription, but still prepares and hashes the audio. Older cache keys
|
|
196
|
+
are invalidated once; malformed entries are ignored and rebuilt.
|
|
187
197
|
3. Both sides are normalised to comparable tokens: case folded, accents stripped, punctuation
|
|
188
198
|
dropped, digits and roman numerals spelled out ("Chapter XVIII" and "chapter eighteen" become
|
|
189
199
|
the same tokens).
|
|
@@ -207,9 +217,11 @@ written; exceeding the threshold only changes the exit code to 3 so a script can
|
|
|
207
217
|
--audio PATH a file, a directory of numbered files, or a glob
|
|
208
218
|
--epub PATH the matching DRM-free EPUB
|
|
209
219
|
--out PATH the EPUB to write
|
|
220
|
+
--report PATH JSON report path (default: readalign-report.json beside --out)
|
|
210
221
|
--model NAME faster-whisper model (default: large-v3-turbo)
|
|
211
222
|
--language CODE ISO code of the narration (default: detect)
|
|
212
223
|
--drift-threshold SECS exit 3 if any sentence drifts more than this (default: 2.5)
|
|
224
|
+
--min-token-match N exit 5 if matched book-token fraction is below N (0 to 1; default: 0)
|
|
213
225
|
--sidecars/--no-sidecars write WebVTT and align.json next to the output (default: on)
|
|
214
226
|
--dry-run probe the inputs and print the plan, transcribe nothing
|
|
215
227
|
--narrator NAME value for the media:narrator metadata
|
|
@@ -278,6 +290,7 @@ against the reference Media Overlays title from the EPUB test suite at
|
|
|
278
290
|
| 2 | the input is DRM-protected |
|
|
279
291
|
| 3 | the EPUB was written but max drift exceeded `--drift-threshold` |
|
|
280
292
|
| 4 | the EPUB was written but epubcheck reported problems (only with `--verify`) |
|
|
293
|
+
| 5 | the EPUB was written but token match was below `--min-token-match` |
|
|
281
294
|
| 130 | interrupted |
|
|
282
295
|
|
|
283
296
|
Failures print one line, not a traceback.
|
|
@@ -10,6 +10,7 @@ import contextlib
|
|
|
10
10
|
import glob
|
|
11
11
|
import hashlib
|
|
12
12
|
import json
|
|
13
|
+
import math
|
|
13
14
|
import os
|
|
14
15
|
import site
|
|
15
16
|
import time
|
|
@@ -17,6 +18,7 @@ from dataclasses import asdict, dataclass
|
|
|
17
18
|
|
|
18
19
|
from .audio import AudioTrack, read_wav_segment
|
|
19
20
|
from .errors import DependencyError
|
|
21
|
+
from .fs import atomic_output
|
|
20
22
|
|
|
21
23
|
_CUDA_READY = False
|
|
22
24
|
|
|
@@ -152,13 +154,17 @@ def _cache_key(
|
|
|
152
154
|
track: AudioTrack, model: str, language: str | None, vad: bool, beam_size: int,
|
|
153
155
|
window_seconds: float,
|
|
154
156
|
) -> str:
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
157
|
+
digest = hashlib.sha256()
|
|
158
|
+
# Size and duration are not identities: edited tracks can retain both.
|
|
159
|
+
with open(track.path, "rb") as handle:
|
|
160
|
+
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
|
161
|
+
digest.update(chunk)
|
|
162
|
+
settings = {"version": 2, "model": model, "language": language, "vad": vad,
|
|
163
|
+
"beam_size": beam_size, "window_seconds": window_seconds,
|
|
164
|
+
"duration": track.duration,
|
|
165
|
+
"chapters": [(chapter.start, chapter.end) for chapter in track.chapters]}
|
|
166
|
+
digest.update(json.dumps(settings, sort_keys=True).encode("utf-8"))
|
|
167
|
+
return digest.hexdigest()
|
|
162
168
|
|
|
163
169
|
|
|
164
170
|
def _load_cached(cache_dir: str | None, key: str) -> tuple[list[Word], str | None] | None:
|
|
@@ -169,13 +175,26 @@ def _load_cached(cache_dir: str | None, key: str) -> tuple[list[Word], str | Non
|
|
|
169
175
|
try:
|
|
170
176
|
with open(path, encoding="utf-8") as handle:
|
|
171
177
|
data = json.load(handle)
|
|
172
|
-
except (OSError,
|
|
178
|
+
except (OSError, ValueError, UnicodeError):
|
|
173
179
|
return None
|
|
174
180
|
try:
|
|
175
181
|
# Caches written before the language was recorded are a bare list of words.
|
|
176
|
-
if isinstance(data, list):
|
|
177
|
-
return
|
|
178
|
-
|
|
182
|
+
if not isinstance(data, (dict, list)):
|
|
183
|
+
return None
|
|
184
|
+
rows = data if isinstance(data, list) else data["words"]
|
|
185
|
+
language = None if isinstance(data, list) else data.get("language")
|
|
186
|
+
if not isinstance(rows, list) or (language is not None and not isinstance(language, str)):
|
|
187
|
+
return None
|
|
188
|
+
words = [Word(**entry) for entry in rows]
|
|
189
|
+
for word in words:
|
|
190
|
+
if not isinstance(word.text, str) or not word.text.strip():
|
|
191
|
+
return None
|
|
192
|
+
values = (word.start, word.end, word.probability)
|
|
193
|
+
if any(type(value) not in (int, float) or not math.isfinite(value) for value in values):
|
|
194
|
+
return None
|
|
195
|
+
if word.start < 0 or word.end < word.start or not 0 <= word.probability <= 1:
|
|
196
|
+
return None
|
|
197
|
+
return words, language
|
|
179
198
|
except (KeyError, TypeError):
|
|
180
199
|
return None
|
|
181
200
|
|
|
@@ -188,7 +207,7 @@ def _store_cached(
|
|
|
188
207
|
try:
|
|
189
208
|
os.makedirs(cache_dir, exist_ok=True)
|
|
190
209
|
path = os.path.join(cache_dir, f"{key}.json")
|
|
191
|
-
with
|
|
210
|
+
with atomic_output(path) as temporary, open(temporary, "w", encoding="utf-8") as handle:
|
|
192
211
|
json.dump({"language": language, "words": [asdict(w) for w in words]}, handle)
|
|
193
212
|
except OSError as exc:
|
|
194
213
|
log(f" warning: could not write the transcript cache: {exc}")
|
|
@@ -220,7 +239,7 @@ def transcribe_tracks(
|
|
|
220
239
|
|
|
221
240
|
for track in tracks:
|
|
222
241
|
key = (
|
|
223
|
-
_cache_key(track, model_name, language, vad, beam_size, window_seconds)
|
|
242
|
+
_cache_key(track, model_name, settings.language, vad, beam_size, window_seconds)
|
|
224
243
|
if cache_dir
|
|
225
244
|
else ""
|
|
226
245
|
)
|
|
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|
|
9
9
|
|
|
10
10
|
import glob as globlib
|
|
11
11
|
import json
|
|
12
|
+
import math
|
|
12
13
|
import os
|
|
13
14
|
import re
|
|
14
15
|
import shutil
|
|
@@ -19,6 +20,7 @@ from dataclasses import dataclass, field
|
|
|
19
20
|
import numpy as np
|
|
20
21
|
|
|
21
22
|
from .errors import DependencyError, DRMError, InputError
|
|
23
|
+
from .fs import same_path
|
|
22
24
|
|
|
23
25
|
AUDIO_EXTENSIONS = {
|
|
24
26
|
".mp3", ".m4a", ".m4b", ".mp4", ".flac", ".ogg", ".oga", ".opus", ".wav", ".aiff", ".aif",
|
|
@@ -117,9 +119,9 @@ def _duration_of(probe: dict, path: str) -> float:
|
|
|
117
119
|
if value:
|
|
118
120
|
try:
|
|
119
121
|
duration = float(value)
|
|
120
|
-
except ValueError:
|
|
122
|
+
except (TypeError, ValueError):
|
|
121
123
|
continue
|
|
122
|
-
if duration > 0:
|
|
124
|
+
if math.isfinite(duration) and duration > 0:
|
|
123
125
|
return duration
|
|
124
126
|
raise InputError(f"{path} has no readable duration; is it a complete audio file?")
|
|
125
127
|
|
|
@@ -190,7 +192,7 @@ def _chapters_from(probe: dict) -> list[Chapter]:
|
|
|
190
192
|
end = float(entry.get("end_time", 0.0))
|
|
191
193
|
except (TypeError, ValueError):
|
|
192
194
|
continue
|
|
193
|
-
if end <= start:
|
|
195
|
+
if not math.isfinite(start) or not math.isfinite(end) or start < 0 or end <= start:
|
|
194
196
|
continue
|
|
195
197
|
title = (entry.get("tags") or {}).get("title") or f"Chapter {len(chapters) + 1}"
|
|
196
198
|
chapters.append(Chapter(start=start, end=end, title=title))
|
|
@@ -228,6 +230,8 @@ def prepare_tracks(sources: list[str], workdir: str, bitrate: str, log=print) ->
|
|
|
228
230
|
_reject_drm(source, probe)
|
|
229
231
|
suffix, media_type, transcode = _output_plan(source, probe)
|
|
230
232
|
target = os.path.join(audio_dir, f"part{index:03d}{suffix}")
|
|
233
|
+
if any(same_path(target, original) for original in sources):
|
|
234
|
+
raise InputError(f"the work directory would overwrite source audio: {target}")
|
|
231
235
|
label = f"[{index}/{len(sources)}] {os.path.basename(source)}"
|
|
232
236
|
log(f" {label} -> {'transcoding to AAC' if transcode else 'stream copy'}")
|
|
233
237
|
command = ["ffmpeg", "-nostdin", "-v", "error", "-y", "-i", source,
|
|
@@ -265,6 +269,8 @@ def decode_wav(track: AudioTrack, workdir: str) -> str:
|
|
|
265
269
|
wav_dir = os.path.join(workdir, "wav")
|
|
266
270
|
os.makedirs(wav_dir, exist_ok=True)
|
|
267
271
|
target = os.path.join(wav_dir, f"part{track.index:03d}.wav")
|
|
272
|
+
if same_path(target, track.source):
|
|
273
|
+
raise InputError(f"the work directory would overwrite source audio: {target}")
|
|
268
274
|
result = _run(
|
|
269
275
|
[
|
|
270
276
|
"ffmpeg", "-nostdin", "-v", "error", "-y", "-i", track.path,
|
|
@@ -8,6 +8,7 @@ from __future__ import annotations
|
|
|
8
8
|
|
|
9
9
|
import argparse
|
|
10
10
|
import json
|
|
11
|
+
import math
|
|
11
12
|
import os
|
|
12
13
|
import posixpath
|
|
13
14
|
import shutil
|
|
@@ -15,10 +16,12 @@ import sys
|
|
|
15
16
|
import tempfile
|
|
16
17
|
import time
|
|
17
18
|
import zipfile
|
|
19
|
+
import zlib
|
|
18
20
|
from urllib.parse import unquote
|
|
19
21
|
|
|
20
22
|
from . import __version__
|
|
21
23
|
from .errors import InputError, ReadAlignError
|
|
24
|
+
from .fs import ensure_directory, same_path
|
|
22
25
|
from .members import open_archive
|
|
23
26
|
|
|
24
27
|
DEFAULT_MODEL = "large-v3-turbo"
|
|
@@ -46,6 +49,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
46
49
|
help="audio file, or a directory of numbered files, or a glob")
|
|
47
50
|
build.add_argument("--epub", required=True, help="the matching DRM-free EPUB")
|
|
48
51
|
build.add_argument("--out", required=True, help="path of the EPUB to write")
|
|
52
|
+
build.add_argument("--report", default=None, metavar="PATH",
|
|
53
|
+
help="JSON report path (default: readalign-report.json beside --out)")
|
|
49
54
|
build.add_argument("--model", default=DEFAULT_MODEL,
|
|
50
55
|
help=f"faster-whisper model (default: {DEFAULT_MODEL})")
|
|
51
56
|
build.add_argument("--language", default=None,
|
|
@@ -53,6 +58,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
53
58
|
build.add_argument("--drift-threshold", type=float, default=DEFAULT_DRIFT, metavar="SECONDS",
|
|
54
59
|
help="fail if any sentence drifts more than this "
|
|
55
60
|
f"(default: {DEFAULT_DRIFT})")
|
|
61
|
+
build.add_argument("--min-token-match", type=float, default=0.0, metavar="FRACTION",
|
|
62
|
+
help="exit 5 if the matched book-token fraction is below this "
|
|
63
|
+
"(0 to 1, default: 0, disabled)")
|
|
56
64
|
sidecars = build.add_mutually_exclusive_group()
|
|
57
65
|
sidecars.add_argument("--sidecars", dest="sidecars", action="store_true", default=True,
|
|
58
66
|
help="write WebVTT and align.json next to the output (default)")
|
|
@@ -134,13 +142,37 @@ def run_build(args) -> int:
|
|
|
134
142
|
started = time.monotonic()
|
|
135
143
|
warnings: list[str] = []
|
|
136
144
|
|
|
137
|
-
if args.drift_threshold <= 0:
|
|
138
|
-
raise InputError("--drift-threshold must be a positive number of seconds")
|
|
145
|
+
if not math.isfinite(args.drift_threshold) or args.drift_threshold <= 0:
|
|
146
|
+
raise InputError("--drift-threshold must be a finite positive number of seconds")
|
|
147
|
+
if not math.isfinite(args.min_token_match) or not 0 <= args.min_token_match <= 1:
|
|
148
|
+
raise InputError("--min-token-match must be a finite fraction between 0 and 1")
|
|
149
|
+
if args.beam_size < 1:
|
|
150
|
+
raise InputError("--beam-size must be a positive integer")
|
|
139
151
|
out_path = os.path.abspath(args.out)
|
|
140
152
|
if os.path.isdir(out_path):
|
|
141
153
|
raise InputError(f"--out points at a directory: {out_path}")
|
|
142
|
-
if out_path
|
|
154
|
+
if same_path(out_path, args.epub):
|
|
143
155
|
raise InputError("--out is the same file as --epub, which would overwrite your ebook")
|
|
156
|
+
report_path = os.path.abspath(args.report) if args.report else os.path.join(
|
|
157
|
+
os.path.dirname(out_path), "readalign-report.json"
|
|
158
|
+
)
|
|
159
|
+
if os.path.isdir(report_path):
|
|
160
|
+
raise InputError(f"--report points at a directory: {report_path}")
|
|
161
|
+
if same_path(report_path, args.epub) or same_path(report_path, out_path):
|
|
162
|
+
raise InputError("--report would overwrite the input or output EPUB")
|
|
163
|
+
directory = os.path.join(os.path.dirname(out_path),
|
|
164
|
+
os.path.splitext(os.path.basename(out_path))[0] + "-sidecars")
|
|
165
|
+
if args.sidecars:
|
|
166
|
+
sidecar_root = os.path.normcase(os.path.realpath(directory))
|
|
167
|
+
report_real = os.path.normcase(os.path.realpath(report_path))
|
|
168
|
+
if report_real == sidecar_root or report_real.startswith(sidecar_root + os.sep):
|
|
169
|
+
raise InputError("--report must be outside the generated sidecar directory")
|
|
170
|
+
|
|
171
|
+
verification_tool = None
|
|
172
|
+
if args.verify and not args.dry_run:
|
|
173
|
+
from .verify import find_epubcheck
|
|
174
|
+
|
|
175
|
+
verification_tool = find_epubcheck(args.verify)
|
|
144
176
|
|
|
145
177
|
require_ffmpeg()
|
|
146
178
|
log(f"readalign {__version__}")
|
|
@@ -155,11 +187,18 @@ def run_build(args) -> int:
|
|
|
155
187
|
log(f" {len(package.docs)} content documents, {len(sentences)} sentences")
|
|
156
188
|
|
|
157
189
|
sources = discover_inputs(args.audio)
|
|
190
|
+
for source in sources:
|
|
191
|
+
if same_path(out_path, source) or same_path(report_path, source):
|
|
192
|
+
raise InputError(f"an output path would overwrite source audio: {source}")
|
|
158
193
|
log(f"preparing {len(sources)} audio file(s)")
|
|
159
194
|
work_dir = args.work_dir or tempfile.mkdtemp(prefix="readalign-")
|
|
160
195
|
temporary = args.work_dir is None
|
|
161
|
-
|
|
196
|
+
ensure_directory(work_dir, "work directory")
|
|
162
197
|
try:
|
|
198
|
+
for index in range(1, len(sources) + 1):
|
|
199
|
+
wav_path = os.path.join(work_dir, "wav", f"part{index:03d}.wav")
|
|
200
|
+
if any(same_path(wav_path, source) for source in [args.epub, *sources]):
|
|
201
|
+
raise InputError(f"the work directory would overwrite an input: {wav_path}")
|
|
163
202
|
tracks = prepare_tracks(sources, work_dir, args.audio_bitrate, log=log)
|
|
164
203
|
if args.dry_run:
|
|
165
204
|
return _dry_run_plan(package, tracks, log)
|
|
@@ -217,11 +256,10 @@ def run_build(args) -> int:
|
|
|
217
256
|
|
|
218
257
|
verification = None
|
|
219
258
|
if args.verify:
|
|
220
|
-
from .verify import
|
|
259
|
+
from .verify import run_epubcheck
|
|
221
260
|
|
|
222
|
-
tool = find_epubcheck(args.verify)
|
|
223
261
|
log("verifying with epubcheck")
|
|
224
|
-
verification = run_epubcheck(out_path,
|
|
262
|
+
verification = run_epubcheck(out_path, verification_tool, log=log)
|
|
225
263
|
|
|
226
264
|
stats = document_stats(package, result)
|
|
227
265
|
for doc in package.docs:
|
|
@@ -241,19 +279,19 @@ def run_build(args) -> int:
|
|
|
241
279
|
drift_threshold=args.drift_threshold,
|
|
242
280
|
warnings=warnings,
|
|
243
281
|
extras={"elapsed_seconds": round(time.monotonic() - started, 1),
|
|
282
|
+
"token_match_gate": {
|
|
283
|
+
"threshold": args.min_token_match,
|
|
284
|
+
"passed": (result.matched_tokens / result.book_tokens
|
|
285
|
+
if result.book_tokens else 0.0) >= args.min_token_match,
|
|
286
|
+
},
|
|
244
287
|
"overlay_duration": summary["total_duration"],
|
|
245
288
|
**({"verify": verification} if verification is not None else {})},
|
|
246
289
|
)
|
|
247
|
-
report_path = os.path.join(os.path.dirname(out_path) or ".", "readalign-report.json")
|
|
248
290
|
write_report(report_path, report)
|
|
249
291
|
|
|
250
292
|
if args.sidecars:
|
|
251
293
|
from .sidecar import write_sidecars
|
|
252
294
|
|
|
253
|
-
directory = os.path.join(
|
|
254
|
-
os.path.dirname(out_path) or ".",
|
|
255
|
-
os.path.splitext(os.path.basename(out_path))[0] + "-sidecars",
|
|
256
|
-
)
|
|
257
295
|
written = write_sidecars(
|
|
258
296
|
directory, result, tracks,
|
|
259
297
|
{index: doc.href for index, doc in enumerate(package.docs)},
|
|
@@ -275,17 +313,25 @@ def run_build(args) -> int:
|
|
|
275
313
|
print(
|
|
276
314
|
f"readalign: max drift {report['drift']['max']:.2f}s exceeds the "
|
|
277
315
|
f"--drift-threshold of {args.drift_threshold:.2f}s. The EPUB was still written; "
|
|
278
|
-
f"check
|
|
316
|
+
f"check {report_path} for where it slipped.",
|
|
279
317
|
file=sys.stderr,
|
|
280
318
|
)
|
|
281
319
|
return 3
|
|
282
320
|
if verification is not None and not verification["ok"]:
|
|
283
321
|
print(
|
|
284
322
|
"readalign: epubcheck reported problems with the output; the lines above are its "
|
|
285
|
-
"own. See the verify section of
|
|
323
|
+
f"own. See the verify section of {report_path}.",
|
|
286
324
|
file=sys.stderr,
|
|
287
325
|
)
|
|
288
326
|
return 4
|
|
327
|
+
if not report["token_match_gate"]["passed"]:
|
|
328
|
+
print(
|
|
329
|
+
f"readalign: token match {report['coverage']['token_match_rate']:.1%} is below "
|
|
330
|
+
f"--min-token-match {args.min_token_match:.1%}. The EPUB was still written; "
|
|
331
|
+
f"inspect {report_path} for unmatched regions.",
|
|
332
|
+
file=sys.stderr,
|
|
333
|
+
)
|
|
334
|
+
return 5
|
|
289
335
|
return 0
|
|
290
336
|
|
|
291
337
|
|
|
@@ -304,7 +350,8 @@ def _open_package(archive, path: str):
|
|
|
304
350
|
if "META-INF/container.xml" not in names:
|
|
305
351
|
raise InputError(f"{path} has no META-INF/container.xml, so it is not an EPUB")
|
|
306
352
|
try:
|
|
307
|
-
container = etree.fromstring(archive.read("META-INF/container.xml")
|
|
353
|
+
container = etree.fromstring(archive.read("META-INF/container.xml"),
|
|
354
|
+
etree.XMLParser(resolve_entities=False))
|
|
308
355
|
except etree.XMLSyntaxError as exc:
|
|
309
356
|
raise InputError(
|
|
310
357
|
f"{path}: META-INF/container.xml is not well-formed XML: {exc}"
|
|
@@ -316,7 +363,8 @@ def _open_package(archive, path: str):
|
|
|
316
363
|
if opf_path not in names:
|
|
317
364
|
raise InputError(f"{path} names a package document that is missing: {opf_path}")
|
|
318
365
|
try:
|
|
319
|
-
return opf_path, etree.fromstring(archive.read(opf_path)
|
|
366
|
+
return opf_path, etree.fromstring(archive.read(opf_path),
|
|
367
|
+
etree.XMLParser(resolve_entities=False))
|
|
320
368
|
except etree.XMLSyntaxError as exc:
|
|
321
369
|
raise InputError(
|
|
322
370
|
f"{path}: the package document {opf_path} is not well-formed XML: {exc}"
|
|
@@ -367,18 +415,19 @@ def _fragment_ids(
|
|
|
367
415
|
def _probe_clip_ends(archive, max_clip_end: dict[str, float], errors: list[str],
|
|
368
416
|
warnings: list[str]) -> dict[str, float]:
|
|
369
417
|
"""Acceptance check: no clipEnd may sit past the real duration ffprobe reports."""
|
|
370
|
-
from .audio import ffprobe
|
|
418
|
+
from .audio import _duration_of, ffprobe
|
|
371
419
|
|
|
372
420
|
durations: dict[str, float] = {}
|
|
373
421
|
with tempfile.TemporaryDirectory(prefix="readalign-check-") as scratch:
|
|
374
|
-
for audio_path, clip_end in sorted(max_clip_end.items()):
|
|
375
|
-
|
|
422
|
+
for index, (audio_path, clip_end) in enumerate(sorted(max_clip_end.items())):
|
|
423
|
+
# Archive names are URL paths, not trusted OS filenames (especially on Windows).
|
|
424
|
+
extracted = os.path.join(scratch, f"audio-{index:04d}")
|
|
376
425
|
with archive.open(audio_path) as source, open(extracted, "wb") as handle:
|
|
377
426
|
shutil.copyfileobj(source, handle, 1 << 20)
|
|
378
427
|
try:
|
|
379
|
-
duration =
|
|
428
|
+
duration = _duration_of(ffprobe(extracted), audio_path)
|
|
380
429
|
except (ReadAlignError, KeyError, ValueError, TypeError) as exc:
|
|
381
|
-
|
|
430
|
+
errors.append(f"could not probe {audio_path}: {exc}")
|
|
382
431
|
continue
|
|
383
432
|
durations[audio_path] = round(duration, 3)
|
|
384
433
|
if clip_end > duration + 0.05:
|
|
@@ -391,6 +440,13 @@ def _probe_clip_ends(archive, max_clip_end: dict[str, float], errors: list[str],
|
|
|
391
440
|
|
|
392
441
|
def check_epub(path: str, *, probe_audio: bool = True) -> dict:
|
|
393
442
|
"""Validate the media overlays of an EPUB. Structural faults are reported, not raised."""
|
|
443
|
+
try:
|
|
444
|
+
return _check_epub(path, probe_audio=probe_audio)
|
|
445
|
+
except (zipfile.BadZipFile, zlib.error, NotImplementedError, RuntimeError) as exc:
|
|
446
|
+
raise InputError(f"{path} is not a readable EPUB: {exc}") from None
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _check_epub(path: str, *, probe_audio: bool) -> dict:
|
|
394
450
|
from lxml import etree
|
|
395
451
|
|
|
396
452
|
from .clock import parse_clock
|
|
@@ -401,6 +457,8 @@ def check_epub(path: str, *, probe_audio: bool = True) -> dict:
|
|
|
401
457
|
warnings: list[str] = []
|
|
402
458
|
with open_archive(path) as archive:
|
|
403
459
|
names = set(archive.namelist())
|
|
460
|
+
if len(names) != len(archive.namelist()):
|
|
461
|
+
errors.append("the archive contains duplicate member names")
|
|
404
462
|
entries = archive.infolist()
|
|
405
463
|
if not entries or entries[0].filename != "mimetype":
|
|
406
464
|
errors.append("the first archive entry is not 'mimetype'")
|
|
@@ -411,6 +469,13 @@ def check_epub(path: str, *, probe_audio: bool = True) -> dict:
|
|
|
411
469
|
|
|
412
470
|
opf_path, opf = _open_package(archive, path)
|
|
413
471
|
items = {item.get("id"): item for item in opf.iter(f"{{{OPF_NS}}}item") if item.get("id")}
|
|
472
|
+
manifest_items = list(opf.iter(f"{{{OPF_NS}}}item"))
|
|
473
|
+
if len(items) != len(manifest_items):
|
|
474
|
+
errors.append("the manifest has missing or duplicate item ids")
|
|
475
|
+
manifested = {
|
|
476
|
+
_resolve(opf_path, item.get("href", "")): item.get("media-type", "")
|
|
477
|
+
for item in items.values()
|
|
478
|
+
}
|
|
414
479
|
spine = [ref.get("idref") for ref in opf.iter(f"{{{OPF_NS}}}itemref") if ref.get("idref")]
|
|
415
480
|
for idref in spine:
|
|
416
481
|
if idref not in items:
|
|
@@ -448,12 +513,17 @@ def check_epub(path: str, *, probe_audio: bool = True) -> dict:
|
|
|
448
513
|
if overlay_id not in durations:
|
|
449
514
|
warnings.append(f"overlay '{overlay_id}' has no media:duration metadata")
|
|
450
515
|
try:
|
|
451
|
-
smil = etree.fromstring(archive.read(smil_path)
|
|
516
|
+
smil = etree.fromstring(archive.read(smil_path),
|
|
517
|
+
etree.XMLParser(resolve_entities=False))
|
|
452
518
|
except etree.XMLSyntaxError as exc:
|
|
453
519
|
errors.append(f"{smil_path}: the overlay is not well-formed XML: {exc}")
|
|
454
520
|
continue
|
|
455
521
|
overlay_clip = 0.0
|
|
456
|
-
|
|
522
|
+
overlay_pars = list(smil.iter(f"{{{SMIL_NS}}}par"))
|
|
523
|
+
if smil.tag != f"{{{SMIL_NS}}}smil" or not overlay_pars:
|
|
524
|
+
errors.append(f"{smil_path}: overlay has no SMIL par elements")
|
|
525
|
+
expected_document = _resolve(opf_path, item.get("href", ""))
|
|
526
|
+
for par in overlay_pars:
|
|
457
527
|
pars += 1
|
|
458
528
|
text = par.find(f"{{{SMIL_NS}}}text")
|
|
459
529
|
audio = par.find(f"{{{SMIL_NS}}}audio")
|
|
@@ -463,6 +533,10 @@ def check_epub(path: str, *, probe_audio: bool = True) -> dict:
|
|
|
463
533
|
src = text.get("src")
|
|
464
534
|
doc_path = _resolve(smil_path, src)
|
|
465
535
|
fragment = unquote(src.split("#", 1)[1]) if "#" in src else ""
|
|
536
|
+
if doc_path != expected_document:
|
|
537
|
+
errors.append(f"{smil_path}: text points outside its associated document")
|
|
538
|
+
if manifested.get(doc_path) not in {"application/xhtml+xml", "image/svg+xml"}:
|
|
539
|
+
errors.append(f"{smil_path}: text target is not a manifested content document")
|
|
466
540
|
if doc_path not in names:
|
|
467
541
|
errors.append(f"{smil_path}: text points at a missing document {doc_path}")
|
|
468
542
|
elif fragment:
|
|
@@ -473,6 +547,8 @@ def check_epub(path: str, *, probe_audio: bool = True) -> dict:
|
|
|
473
547
|
errors.append(f"{smil_path}: a par has no audio src")
|
|
474
548
|
continue
|
|
475
549
|
audio_path = _resolve(smil_path, audio.get("src"))
|
|
550
|
+
if not manifested.get(audio_path, "").startswith("audio/"):
|
|
551
|
+
errors.append(f"{smil_path}: audio target is not a manifested audio file")
|
|
476
552
|
if audio_path not in names:
|
|
477
553
|
errors.append(f"{smil_path}: audio points at a missing file {audio_path}")
|
|
478
554
|
continue
|
|
@@ -489,7 +565,7 @@ def check_epub(path: str, *, probe_audio: bool = True) -> dict:
|
|
|
489
565
|
clip_total += overlay_clip
|
|
490
566
|
declared = durations.get(overlay_id)
|
|
491
567
|
if declared is not None and abs(declared - overlay_clip) > 1.0:
|
|
492
|
-
|
|
568
|
+
errors.append(
|
|
493
569
|
f"overlay '{overlay_id}' declares {declared:.3f}s but its clips total "
|
|
494
570
|
f"{overlay_clip:.3f}s"
|
|
495
571
|
)
|
|
@@ -503,6 +579,11 @@ def check_epub(path: str, *, probe_audio: bool = True) -> dict:
|
|
|
503
579
|
errors.append("no spine document has a media-overlay attribute")
|
|
504
580
|
if total_meta is None and overlays:
|
|
505
581
|
warnings.append("the package has no unrefined media:duration for the whole book")
|
|
582
|
+
elif total_meta is not None and abs(total_meta - clip_total) > 1.0:
|
|
583
|
+
errors.append(
|
|
584
|
+
f"package media:duration declares {total_meta:.3f}s but its clips total "
|
|
585
|
+
f"{clip_total:.3f}s"
|
|
586
|
+
)
|
|
506
587
|
|
|
507
588
|
return {
|
|
508
589
|
"epub": os.path.abspath(path),
|