sci2audio 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. sci2audio/__init__.py +3 -0
  2. sci2audio/app/__init__.py +110 -0
  3. sci2audio/app/audio.py +847 -0
  4. sci2audio/app/chapters.py +109 -0
  5. sci2audio/app/chunks.py +102 -0
  6. sci2audio/app/config.py +136 -0
  7. sci2audio/app/estimate.py +192 -0
  8. sci2audio/app/files.py +98 -0
  9. sci2audio/app/job.py +285 -0
  10. sci2audio/app/lock.py +92 -0
  11. sci2audio/app/report.py +400 -0
  12. sci2audio/app/runner.py +92 -0
  13. sci2audio/app/script.py +95 -0
  14. sci2audio/app/settings.py +58 -0
  15. sci2audio/app/voice.py +70 -0
  16. sci2audio/cli.py +1013 -0
  17. sci2audio/core/__init__.py +81 -0
  18. sci2audio/core/audio.py +39 -0
  19. sci2audio/core/content.py +266 -0
  20. sci2audio/core/document.py +225 -0
  21. sci2audio/core/edits.py +121 -0
  22. sci2audio/core/export.py +76 -0
  23. sci2audio/extraction/__init__.py +28 -0
  24. sci2audio/extraction/cover.py +68 -0
  25. sci2audio/extraction/epub.py +576 -0
  26. sci2audio/extraction/errors.py +7 -0
  27. sci2audio/extraction/markdown.py +250 -0
  28. sci2audio/extraction/markup.py +356 -0
  29. sci2audio/extraction/pdf.py +623 -0
  30. sci2audio/extraction/text.py +136 -0
  31. sci2audio/layout/__init__.py +13 -0
  32. sci2audio/layout/build.py +265 -0
  33. sci2audio/layout/captions.py +77 -0
  34. sci2audio/layout/footnotes.py +180 -0
  35. sci2audio/layout/formulas.py +392 -0
  36. sci2audio/layout/furniture.py +326 -0
  37. sci2audio/layout/geometry.py +124 -0
  38. sci2audio/layout/graphics.py +569 -0
  39. sci2audio/layout/headings.py +358 -0
  40. sci2audio/layout/lists.py +167 -0
  41. sci2audio/layout/maths.py +47 -0
  42. sci2audio/layout/order.py +374 -0
  43. sci2audio/layout/paragraphs.py +518 -0
  44. sci2audio/layout/settings.py +142 -0
  45. sci2audio/layout/split.py +113 -0
  46. sci2audio/layout/tagged.py +442 -0
  47. sci2audio/output/__init__.py +44 -0
  48. sci2audio/output/ffmpeg.py +111 -0
  49. sci2audio/output/model.py +125 -0
  50. sci2audio/output/names.py +70 -0
  51. sci2audio/output/readalong.py +160 -0
  52. sci2audio/output/timeline.py +113 -0
  53. sci2audio/output/wav.py +83 -0
  54. sci2audio/output/write.py +280 -0
  55. sci2audio/testing/__init__.py +32 -0
  56. sci2audio/testing/documents.py +124 -0
  57. sci2audio/testing/regression.py +197 -0
  58. sci2audio/testing/text_steps.py +262 -0
  59. sci2audio/testing/voices.py +164 -0
  60. sci2audio/textprep/__init__.py +105 -0
  61. sci2audio/textprep/abbreviations.py +138 -0
  62. sci2audio/textprep/characters.py +267 -0
  63. sci2audio/textprep/citations.py +683 -0
  64. sci2audio/textprep/data/en/abbreviations.toml +209 -0
  65. sci2audio/textprep/data/en/units.toml +548 -0
  66. sci2audio/textprep/footnotes.py +418 -0
  67. sci2audio/textprep/furniture.py +46 -0
  68. sci2audio/textprep/hyphens.py +228 -0
  69. sci2audio/textprep/links.py +628 -0
  70. sci2audio/textprep/notation.py +170 -0
  71. sci2audio/textprep/numbers.py +529 -0
  72. sci2audio/textprep/placeholders.py +68 -0
  73. sci2audio/textprep/references.py +107 -0
  74. sci2audio/textprep/registry.py +83 -0
  75. sci2audio/textprep/runner.py +206 -0
  76. sci2audio/textprep/spoken.py +33 -0
  77. sci2audio/textprep/statistics.py +322 -0
  78. sci2audio/textprep/steps.py +153 -0
  79. sci2audio/textprep/units.py +449 -0
  80. sci2audio/textprep/utterances.py +315 -0
  81. sci2audio/textprep/wording.py +93 -0
  82. sci2audio/textprep/wordlists.py +189 -0
  83. sci2audio/voices/__init__.py +102 -0
  84. sci2audio/voices/download.py +340 -0
  85. sci2audio/voices/engine.py +136 -0
  86. sci2audio/voices/errors.py +88 -0
  87. sci2audio/voices/kokoro_engine.py +346 -0
  88. sci2audio/voices/runtime.py +37 -0
  89. sci2audio/voices/types.py +206 -0
  90. sci2audio/voices/wav.py +82 -0
  91. sci2audio/web/__init__.py +13 -0
  92. sci2audio/web/api.py +658 -0
  93. sci2audio/web/config.py +97 -0
  94. sci2audio/web/documents.py +496 -0
  95. sci2audio/web/i18n.py +188 -0
  96. sci2audio/web/jobs.py +265 -0
  97. sci2audio/web/launch.py +121 -0
  98. sci2audio/web/limits.py +64 -0
  99. sci2audio/web/models.py +178 -0
  100. sci2audio/web/pages.py +509 -0
  101. sci2audio/web/player.py +549 -0
  102. sci2audio/web/security.py +194 -0
  103. sci2audio/web/server.py +116 -0
  104. sci2audio/web/state.py +88 -0
  105. sci2audio/web/static/app.css +150 -0
  106. sci2audio/web/static/app.js +499 -0
  107. sci2audio/web/static/player.js +277 -0
  108. sci2audio/web/strings/de.toml +348 -0
  109. sci2audio/web/strings/en.toml +343 -0
  110. sci2audio/web/tasks.py +280 -0
  111. sci2audio/web/templates/_remove.html +17 -0
  112. sci2audio/web/templates/base.html +32 -0
  113. sci2audio/web/templates/document.html +153 -0
  114. sci2audio/web/templates/index.html +41 -0
  115. sci2audio/web/templates/job.html +66 -0
  116. sci2audio/web/templates/listen.html +75 -0
  117. sci2audio/web/templates/missing.html +7 -0
  118. sci2audio/web/templates/privacy.html +47 -0
  119. sci2audio/web/templates/voice.html +31 -0
  120. sci2audio-1.0.0.dist-info/METADATA +171 -0
  121. sci2audio-1.0.0.dist-info/RECORD +125 -0
  122. sci2audio-1.0.0.dist-info/WHEEL +4 -0
  123. sci2audio-1.0.0.dist-info/entry_points.txt +2 -0
  124. sci2audio-1.0.0.dist-info/licenses/LICENSE +674 -0
  125. sci2audio-1.0.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +86 -0
sci2audio/app/audio.py ADDED
@@ -0,0 +1,847 @@
1
+ """The audio job: a file in, the audiobook out (#71, WI-02, WI-04, VA-04; closes #10).
2
+
3
+ file → script preparation (as the script job, in memory) → the voice speaks every
4
+ utterance of the chosen chapters into the working folder's chunk cache → output writes one
5
+ MP3 per chapter, the M4B and the read-along file → the job's records.
6
+
7
+ - Writes only the audiobook files into the output folder (D-154); the script and the
8
+ report are the script job's.
9
+ - Resume (D-155, D-156): a chunk is cached under a key of its text, voice, speed and engine
10
+ version; running the job again on the same working folder speaks only what is missing.
11
+ All chapters are encoded again at the end (seconds, against minutes of speech).
12
+ - Pauses (D-116, D-157): the app gives every chunk one `Pause` kind; output turns kinds
13
+ into seconds and opens every chapter with the chapter pause.
14
+ - Progress and the time estimate (D-159, D-380 to D-382, D-384, `estimate.py`) are
15
+ measured on this run's own speech; cancel (D-160) by a `threading.Event`; one job per
16
+ working folder (D-161).
17
+ - It never stops to ask (D-61 (3)): an utterance the voice fails on is left out and
18
+ listed, five failures in a row stop the job (D-162). It fails with a `JobError` a person
19
+ can act on, and nothing half-written, when it cannot go on (D-140): the existing
20
+ audiobook without `overwrite` (checked before any speech, D-163), missing voice files,
21
+ a full disk, a removed output folder, ffmpeg failing.
22
+ - The cover (D-313): the file's own cover image, written to the working folder as
23
+ `cover.jpg` or `cover.png` and decoded once by ffmpeg before any speech, goes into every
24
+ MP3 and the M4B. A file without one, or a cover that cannot be read, decoded or
25
+ written, gives an audiobook without a cover, never a failed job.
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import contextlib
31
+ import threading
32
+ import time
33
+ import wave
34
+ from collections.abc import Callable, Iterable, Iterator, Sequence
35
+ from dataclasses import dataclass, field, replace
36
+ from pathlib import Path
37
+ from typing import Any
38
+
39
+ from sci2audio import __version__
40
+ from sci2audio.app.chapters import document_title
41
+ from sci2audio.app.chunks import (
42
+ cache_size,
43
+ chunk_file,
44
+ chunk_key,
45
+ is_finished,
46
+ remove_partials,
47
+ )
48
+ from sci2audio.app.estimate import Estimate
49
+ from sci2audio.app.files import write_bytes_atomically, write_texts
50
+ from sci2audio.app.job import (
51
+ COVERS,
52
+ JobCancelledError,
53
+ JobError,
54
+ OutputExistsError,
55
+ Phase,
56
+ check_folders,
57
+ check_readable,
58
+ json_text,
59
+ read_document,
60
+ settings_data,
61
+ )
62
+ from sci2audio.app.lock import work_folder_lock
63
+ from sci2audio.app.report import Pages
64
+ from sci2audio.app.script import ScriptResult, prepare_script
65
+ from sci2audio.app.settings import JobSettings
66
+ from sci2audio.app.voice import VoiceSource
67
+ from sci2audio.core import AudioChunk, BlockKind, Document, Span
68
+ from sci2audio.core.export import to_data
69
+ from sci2audio.output import (
70
+ BookInfo,
71
+ Chapter,
72
+ ChunkAudioError,
73
+ Ffmpeg,
74
+ FfmpegError,
75
+ FfmpegNotFoundError,
76
+ OutputSettings,
77
+ Pause,
78
+ Spoken,
79
+ Written,
80
+ find_ffmpeg,
81
+ names,
82
+ read_readalong,
83
+ write_audiobook,
84
+ )
85
+ from sci2audio.output.ffmpeg import LOCAL_ONLY
86
+ from sci2audio.textprep import BUILT_IN_STEPS, DEFAULT_MAX_CHARS, TextStep, Utterance
87
+ from sci2audio.voices import (
88
+ Engine,
89
+ EngineError,
90
+ EngineSession,
91
+ ModelNotFoundError,
92
+ speak_to_wav,
93
+ )
94
+
95
+ RECORD = "audio.json"
96
+ RECORD_FORMAT = "sci2audio-audio"
97
+ RECORD_VERSION = 1
98
+ DOCUMENT = "document.json"
99
+ COVER_NAMES = ("cover.jpg", "cover.png") # the cover in the working folder (D-313)
100
+ ESTIMATE_MIN_CHARACTERS = 1000 # measured before the first estimate (D-159, D-382)
101
+ WARM_UP_CALLS = 4 # the voice's first calls are slower; they are not measured (D-381)
102
+ FAILURES_IN_A_ROW = 5 # the voice is taken as broken after this many (D-162)
103
+
104
+
105
+ @dataclass(frozen=True)
106
+ class PlannedChunk:
107
+ """One utterance the voice speaks: its chapter, cache key and the pause before it."""
108
+
109
+ utterance: Utterance
110
+ chapter: int # index in the job's chapter list
111
+ key: str
112
+ pause: Pause
113
+
114
+ @property
115
+ def file(self) -> str:
116
+ return chunk_file(self.key)
117
+
118
+
119
+ @dataclass(frozen=True)
120
+ class AudioPlan:
121
+ """Every chunk of the chosen chapters in order, and the chosen chapters with nothing
122
+ to say (they get no audio)."""
123
+
124
+ chunks: tuple[PlannedChunk, ...]
125
+ silent: tuple[int, ...]
126
+
127
+
128
+ @dataclass(frozen=True)
129
+ class SkippedUtterance:
130
+ """An utterance the voice failed on: left out of the audiobook and listed (D-162)."""
131
+
132
+ chapter: int
133
+ span: Span
134
+ page: int | None
135
+ printed_page: str
136
+ text: str
137
+ error: str
138
+
139
+
140
+ @dataclass(frozen=True)
141
+ class AudioProgress:
142
+ """What the audio job is doing; sent as each phase starts and after every chunk.
143
+
144
+ `characters_*` count only what the voice speaks in this run (cached chunks are not
145
+ spoken again). `seconds_left` is None while the speed is being measured (D-159).
146
+ """
147
+
148
+ phase: Phase
149
+ chapter: int | None = None # the chunk's chapter, index in the job's chapter list
150
+ chapter_title: str = ""
151
+ chapters: int = 0 # chapters with something to say
152
+ chunks_done: int = 0
153
+ chunks_total: int = 0
154
+ reused: int = 0 # chunks found finished in the cache
155
+ characters_done: int = 0
156
+ characters_total: int = 0
157
+ characters_per_second: float | None = None
158
+ seconds_left: float | None = None
159
+
160
+
161
+ @dataclass(frozen=True)
162
+ class AudioJob:
163
+ """What a finished audio job worked out and wrote."""
164
+
165
+ result: ScriptResult
166
+ written: Written
167
+ spoken: int # chunks the voice spoke in this run
168
+ reused: int # chunks taken from the cache (or spoken earlier in this run)
169
+ skipped: tuple[SkippedUtterance, ...]
170
+ duration: float # seconds of the M4B timeline
171
+ characters_per_second: float | None
172
+ record: Path
173
+ document_file: Path
174
+
175
+
176
+ # ---- planning (pure) -----------------------------------------------------------------
177
+
178
+
179
+ def plan_audio(
180
+ result: ScriptResult, *, engine: str, engine_version: str, voice: str, speed: float
181
+ ) -> AudioPlan:
182
+ """The chunks of the chosen chapters with their pause kinds (D-116, D-157):
183
+ PARAGRAPH for the first chunk of another block (HEADING after a heading), SENTENCE for
184
+ the next sentence of one block, NONE where the length limit cut a sentence, and at
185
+ least PAUSE_EDIT after a `pause` edit. A chapter's first chunk gets the chapter pause
186
+ from output, whatever its kind."""
187
+ blocks = result.document.blocks
188
+ chunks: list[PlannedChunk] = []
189
+ silent: list[int] = []
190
+ for chapter in result.chapters:
191
+ if not chapter.selected:
192
+ continue
193
+ previous: int | None = None # block of the previous spoken utterance
194
+ pause_edit = False
195
+ before = len(chunks)
196
+ for utterance in chapter.utterances:
197
+ if utterance.pause:
198
+ pause_edit = True
199
+ continue
200
+ block = utterance.span.block
201
+ if previous is None or block != previous:
202
+ kind = (
203
+ Pause.HEADING
204
+ if previous is not None and blocks[previous].kind is BlockKind.HEADING
205
+ else Pause.PARAGRAPH
206
+ )
207
+ elif utterance.continues:
208
+ kind = Pause.NONE
209
+ else:
210
+ kind = Pause.SENTENCE
211
+ if pause_edit:
212
+ kind = Pause.strongest(Pause.PAUSE_EDIT, kind)
213
+ key = chunk_key(utterance.text, voice, speed, engine, engine_version)
214
+ chunks.append(PlannedChunk(utterance, chapter.chapter.index, key, kind))
215
+ previous, pause_edit = block, False
216
+ if len(chunks) == before:
217
+ silent.append(chapter.chapter.index)
218
+ return AudioPlan(tuple(chunks), tuple(silent))
219
+
220
+
221
+ def assemble(
222
+ plan: AudioPlan,
223
+ durations: dict[str, float],
224
+ titles: dict[int, str],
225
+ ) -> tuple[list[Chapter], list[Spoken]]:
226
+ """Output's chapters and spoken chunks from the plan, leaving out every chunk without
227
+ audio (a skipped utterance); its pause carries over to the next chunk, so a skipped
228
+ paragraph start still gives the paragraph pause."""
229
+ chapters: list[Chapter] = []
230
+ spoken: list[Spoken] = []
231
+ carried = Pause.NONE
232
+ current: int | None = None
233
+ for chunk in plan.chunks:
234
+ if chunk.chapter != current:
235
+ current, carried = None, Pause.NONE
236
+ if chunk.key not in durations:
237
+ carried = Pause.strongest(carried, chunk.pause)
238
+ continue
239
+ if current is None:
240
+ current = chunk.chapter
241
+ chapters.append(Chapter(titles[chunk.chapter], len(spoken)))
242
+ audio = AudioChunk(chunk.utterance.span, chunk.file, durations[chunk.key])
243
+ spoken.append(Spoken(audio, Pause.strongest(carried, chunk.pause)))
244
+ carried = Pause.NONE
245
+ return chapters, spoken
246
+
247
+
248
+ def book_info(document: Document) -> BookInfo:
249
+ """The book's title, authors and language for the files' metadata (the cover is
250
+ `write_cover`'s)."""
251
+ return BookInfo(
252
+ title=document_title(document),
253
+ authors=tuple(a for a in document.content.metadata.authors if a.strip()),
254
+ language=document.language or "en",
255
+ )
256
+
257
+
258
+ def write_cover(source: Path, work: Path) -> Path | None:
259
+ """The source's cover image written to the working folder, or None (D-313).
260
+
261
+ None for a file type or file without a usable cover, and for a cover that cannot be
262
+ read or written: a missing cover never stops a job. A cover left by an earlier run is
263
+ removed first, so the audiobook never shows the cover of another file.
264
+ """
265
+ for name in COVER_NAMES:
266
+ with contextlib.suppress(OSError):
267
+ (work / name).unlink(missing_ok=True)
268
+ find = COVERS.get(source.suffix.lower())
269
+ if find is None:
270
+ return None
271
+ try:
272
+ cover = find(source)
273
+ except Exception: # the cover functions give None on any damage; a bug must not stop the job
274
+ return None
275
+ if cover is None:
276
+ return None
277
+ target = work / f"cover{cover.suffix}"
278
+ try:
279
+ write_bytes_atomically(target, cover.data, overwrite=True)
280
+ except OSError:
281
+ return None
282
+ return target
283
+
284
+
285
+ def usable_cover(cover: Path | None, exe: Path) -> Path | None:
286
+ """`cover` if ffmpeg can decode it, else None and the file removed (D-313).
287
+
288
+ `Cover` checks only the first bytes; a damaged image behind a good signature would make
289
+ ffmpeg refuse the first MP3 after all the speech. Decoding its first frame now, before
290
+ any speech, finds that out in a fraction of a second.
291
+ """
292
+ if cover is None:
293
+ return None
294
+ try:
295
+ Ffmpeg(exe).run([*LOCAL_ONLY, "-i", str(cover), "-frames:v", "1", "-f", "null", "-"])
296
+ except (FfmpegError, OSError):
297
+ with contextlib.suppress(OSError):
298
+ cover.unlink(missing_ok=True)
299
+ return None
300
+ return cover
301
+
302
+
303
+ def audiobook_files(
304
+ out_folder: Path, book: BookInfo, titles: Sequence[str], settings: OutputSettings
305
+ ) -> list[Path]:
306
+ """The files `write_audiobook` writes for these chapters (output's naming rules)."""
307
+ base = names.book_name(book.title)
308
+ files = []
309
+ if settings.mp3:
310
+ count = len(titles)
311
+ files += [
312
+ out_folder / names.chapter_file(base, n, count, title)
313
+ for n, title in enumerate(titles, start=1)
314
+ ]
315
+ if settings.m4b:
316
+ files.append(out_folder / f"{base}.m4b")
317
+ files.append(out_folder / f"{base}.readalong.json")
318
+ return files
319
+
320
+
321
+ # ---- running -------------------------------------------------------------------------
322
+
323
+
324
+ _Identity = tuple[int, int, int, int] # device, inode, size, modification time (ns)
325
+
326
+
327
+ @dataclass(frozen=True)
328
+ class _CancellableFfmpeg(Ffmpeg):
329
+ """Runs the given ffmpeg, stopping the feed when the job is cancelled; output then
330
+ removes its partial file (D-119). Remembers the identity of every file ffmpeg finished
331
+ (device, inode, size and modification time, which the rename into place keeps), so a
332
+ failed run can tell its own files from anyone else's."""
333
+
334
+ inner: Ffmpeg | None = None
335
+ cancel: threading.Event | None = None
336
+ finished: list[_Identity] = field(default_factory=list)
337
+
338
+ def run(self, args: Sequence[str], feed: Iterable[bytes] = ()) -> str:
339
+ runner = self.inner or Ffmpeg(self.exe)
340
+ printed = runner.run(args, self._watched(feed))
341
+ with contextlib.suppress(OSError): # the file it wrote is its last argument
342
+ self.finished.append(_identity(Path(args[-1])))
343
+ return printed
344
+
345
+ def _watched(self, feed: Iterable[bytes]) -> Iterator[bytes]:
346
+ for piece in feed:
347
+ _check_cancel(self.cancel)
348
+ yield piece
349
+
350
+
351
+ def _check_cancel(cancel: threading.Event | None) -> None:
352
+ if cancel is not None and cancel.is_set():
353
+ raise JobCancelledError("the job was cancelled; finished parts are kept for a rerun")
354
+
355
+
356
+ @dataclass
357
+ class _Speech:
358
+ """The speaking phase's counters and the speed measurement (D-159)."""
359
+
360
+ clock: Callable[[], float]
361
+ chunks_total: int
362
+ characters_total: int
363
+ chapters: int
364
+ last_of_chapter: dict[int, int] # chapter -> index of its last chunk
365
+ chunks_done: int = 0
366
+ reused: int = 0
367
+ spoken: int = 0
368
+ characters_done: int = 0
369
+ estimate: Estimate = field(default_factory=Estimate)
370
+ calls: int = 0 # voice calls that spoke a chunk in this run
371
+ first_chapter: int | None = None # chapter of the first chunk spoken in this run
372
+ first_chapter_done: bool = False
373
+ durations: dict[str, float] = field(default_factory=dict)
374
+ failed: dict[str, str] = field(default_factory=dict)
375
+ failures_in_a_row: int = 0
376
+
377
+ def speed(self) -> float | None:
378
+ return self.estimate.speed()
379
+
380
+ def seconds_left(self) -> float | None:
381
+ """The cost model summed over the chunks still to speak (D-380, D-384); None
382
+ while the first chapter is spoken and until enough characters are measured (D-159,
383
+ D-382)."""
384
+ if (
385
+ not self.first_chapter_done
386
+ or self.estimate.measured_characters < ESTIMATE_MIN_CHARACTERS
387
+ ):
388
+ return None
389
+ return self.estimate.seconds_left()
390
+
391
+ def progress(self, chunk: PlannedChunk, title: str) -> AudioProgress:
392
+ return AudioProgress(
393
+ Phase.SPEAKING,
394
+ chapter=chunk.chapter,
395
+ chapter_title=title,
396
+ chapters=self.chapters,
397
+ chunks_done=self.chunks_done,
398
+ chunks_total=self.chunks_total,
399
+ reused=self.reused,
400
+ characters_done=self.characters_done,
401
+ characters_total=self.characters_total,
402
+ characters_per_second=self.speed(),
403
+ seconds_left=self.seconds_left(),
404
+ )
405
+
406
+
407
+ def _cached_duration(path: Path) -> float:
408
+ with wave.open(str(path), "rb") as audio:
409
+ return audio.getnframes() / audio.getframerate()
410
+
411
+
412
+ def run_audio_job(
413
+ source: Path,
414
+ settings: JobSettings,
415
+ work: Path,
416
+ *,
417
+ voice: VoiceSource,
418
+ output_settings: OutputSettings | None = None,
419
+ overwrite: bool = False,
420
+ progress: Callable[[AudioProgress], None] | None = None,
421
+ cancel: threading.Event | None = None,
422
+ steps: Sequence[TextStep] = BUILT_IN_STEPS,
423
+ max_chars: int = DEFAULT_MAX_CHARS,
424
+ ffmpeg: Ffmpeg | None = None,
425
+ clock: Callable[[], float] = time.perf_counter,
426
+ ) -> AudioJob:
427
+ """Turn `source` into an audiobook in the output folder; see the module docstring.
428
+
429
+ `work` is the job's own working folder (made if missing; it holds the chunk cache and
430
+ the records). Existing audiobook files are replaced only with `overwrite`, checked
431
+ before any speech. Raises JobError (or a subclass) when the job cannot go on;
432
+ JobCancelledError when `cancel` is set. Nothing half-written is left.
433
+ """
434
+ tell: Callable[[AudioProgress], None] = progress or (lambda event: None)
435
+ output_settings = output_settings or OutputSettings()
436
+ voice_id = settings.voice or voice.default_voice
437
+ if voice_id not in voice.voices:
438
+ raise JobError(f"there is no {voice.engine} voice {voice_id!r}; choose another voice")
439
+ check_readable(source)
440
+ out_folder = (settings.output_folder or source.absolute().parent).absolute()
441
+ check_folders((out_folder, work))
442
+ _check_cancel(cancel)
443
+ with work_folder_lock(work):
444
+ document = read_document(source, settings, lambda phase: tell(AudioProgress(phase)))
445
+ tell(AudioProgress(Phase.TEXT_STEPS))
446
+ result = prepare_script(document, settings, steps, max_chars)
447
+ plan = plan_audio(
448
+ result,
449
+ engine=voice.engine,
450
+ engine_version=voice.version,
451
+ voice=voice_id,
452
+ speed=settings.speed,
453
+ )
454
+ if not plan.chunks:
455
+ raise JobError("there is nothing to read in the chosen chapters")
456
+ titles = {chapter.chapter.index: chapter.chapter.title for chapter in result.chapters}
457
+ book = book_info(document)
458
+ _check_targets(out_folder, book, plan, titles, output_settings, overwrite)
459
+ try:
460
+ run = _CancellableFfmpeg(
461
+ (ffmpeg or Ffmpeg(find_ffmpeg())).exe, inner=ffmpeg, cancel=cancel
462
+ )
463
+ except FfmpegNotFoundError as error:
464
+ raise JobError(str(error)) from error
465
+ book = replace(book, cover=usable_cover(write_cover(source, work), run.exe))
466
+ _check_cancel(cancel)
467
+
468
+ speech = _speak(plan, work, voice, voice_id, settings.speed, titles, tell, cancel, clock)
469
+ chapters, spoken = assemble(plan, speech.durations, titles)
470
+ if not spoken:
471
+ raise JobError("the voice could not speak any of the chosen chapters")
472
+
473
+ tell(AudioProgress(Phase.ENCODING, chapters=len(chapters)))
474
+ _remove_old_records(work)
475
+ written = _encode(work, out_folder, chapters, spoken, book, output_settings, run, overwrite)
476
+ skipped = _skipped(result, plan, speech)
477
+ record = _record(result, settings, voice, voice_id, plan, speech, skipped, written, work)
478
+ files = _write_records(work, result.document, record)
479
+ tell(AudioProgress(Phase.DONE, chapters=len(chapters)))
480
+ return AudioJob(
481
+ result=result,
482
+ written=written,
483
+ spoken=speech.spoken,
484
+ reused=speech.reused,
485
+ skipped=skipped,
486
+ duration=record["duration"],
487
+ characters_per_second=speech.speed(),
488
+ record=files[0],
489
+ document_file=files[1],
490
+ )
491
+
492
+
493
+ def _check_targets(
494
+ out_folder: Path,
495
+ book: BookInfo,
496
+ plan: AudioPlan,
497
+ titles: dict[int, str],
498
+ settings: OutputSettings,
499
+ overwrite: bool,
500
+ ) -> None:
501
+ """Refuse, before any speech, to replace an existing audiobook without `overwrite`
502
+ (D-163; output checks again when it writes, D-119)."""
503
+ if overwrite:
504
+ return
505
+ chapters = list(dict.fromkeys(chunk.chapter for chunk in plan.chunks))
506
+ targets = audiobook_files(out_folder, book, [titles[c] for c in chapters], settings)
507
+ if existing := [path.name for path in targets if path.exists()]:
508
+ raise OutputExistsError(
509
+ f"these files exist already: {', '.join(existing)}; choose to replace them"
510
+ )
511
+
512
+
513
+ def _speak(
514
+ plan: AudioPlan,
515
+ work: Path,
516
+ voice: VoiceSource,
517
+ voice_id: str,
518
+ speed: float,
519
+ titles: dict[int, str],
520
+ tell: Callable[[AudioProgress], None],
521
+ cancel: threading.Event | None,
522
+ clock: Callable[[], float],
523
+ ) -> _Speech:
524
+ """Speak every chunk not yet in the cache; the engine is made only if one is missing."""
525
+ try:
526
+ remove_partials(work)
527
+ needed: dict[str, int] = {} # key -> characters, in order
528
+ for chunk in plan.chunks:
529
+ if chunk.key not in needed and not is_finished(work / chunk.file):
530
+ needed[chunk.key] = len(chunk.utterance.text)
531
+ except OSError as error:
532
+ raise JobError(f"could not read the working folder: {error.strerror or error}") from error
533
+ last_of_chapter = {chunk.chapter: index for index, chunk in enumerate(plan.chunks)}
534
+ speech = _Speech(
535
+ clock=clock,
536
+ chunks_total=len(plan.chunks),
537
+ characters_total=sum(needed.values()),
538
+ chapters=len(last_of_chapter),
539
+ last_of_chapter=last_of_chapter,
540
+ )
541
+ for characters in needed.values():
542
+ speech.estimate.expect(characters)
543
+ engine: Engine | None = None
544
+ session: EngineSession | None = None
545
+ try:
546
+ if needed:
547
+ tell(AudioProgress(Phase.LOADING_VOICE, chapters=speech.chapters))
548
+ engine, session = _load(voice)
549
+ tell(
550
+ AudioProgress(
551
+ Phase.SPEAKING,
552
+ chapters=speech.chapters,
553
+ chunks_total=speech.chunks_total,
554
+ characters_total=speech.characters_total,
555
+ )
556
+ )
557
+ for index, chunk in enumerate(plan.chunks):
558
+ _check_cancel(cancel)
559
+ _one(chunk, work, needed, session, voice_id, speed, speech)
560
+ speech.chunks_done += 1
561
+ if speech.first_chapter is not None and index == last_of_chapter[speech.first_chapter]:
562
+ speech.first_chapter_done = True
563
+ tell(speech.progress(chunk, titles[chunk.chapter]))
564
+ finally:
565
+ if session is not None:
566
+ session.dispose()
567
+ if engine is not None:
568
+ voice.release(engine)
569
+ return speech
570
+
571
+
572
+ def _load(voice: VoiceSource) -> tuple[Engine, EngineSession]:
573
+ try:
574
+ engine = voice.make()
575
+ except ModelNotFoundError as error:
576
+ raise JobError(f"the voice is not set up: {error}") from error
577
+ except EngineError as error:
578
+ raise JobError(f"the voice could not be loaded: {error}") from error
579
+ try:
580
+ return engine, engine.createSession()
581
+ except EngineError as error:
582
+ voice.release(engine)
583
+ raise JobError(f"the voice could not be started: {error}") from error
584
+
585
+
586
+ def _one(
587
+ chunk: PlannedChunk,
588
+ work: Path,
589
+ needed: dict[str, int],
590
+ session: EngineSession | None,
591
+ voice_id: str,
592
+ speed: float,
593
+ speech: _Speech,
594
+ ) -> None:
595
+ """Speak one chunk, or take it from the cache or from earlier in this run."""
596
+ key, path = chunk.key, work / chunk.file
597
+ if key in speech.durations or key in speech.failed:
598
+ speech.reused += key in speech.durations
599
+ return
600
+ if key not in needed:
601
+ try:
602
+ speech.durations[key] = _cached_duration(path)
603
+ except (OSError, EOFError, wave.Error) as error:
604
+ raise JobError(f"a cached audio chunk became unreadable: {path.name}") from error
605
+ speech.reused += 1
606
+ return
607
+ assert session is not None # noqa: S101 (made whenever a chunk is needed)
608
+ text = chunk.utterance.text
609
+ started = speech.clock()
610
+ try:
611
+ path.parent.mkdir(parents=True, exist_ok=True)
612
+ seconds = speak_to_wav(session, text, voice_id, path, speed=speed)
613
+ except ModelNotFoundError as error:
614
+ raise JobError(f"the voice is not set up: {error}") from error
615
+ except EngineError as error:
616
+ speech.failed[key] = str(error) or type(error).__name__
617
+ speech.characters_done += len(text)
618
+ speech.estimate.done(len(text))
619
+ speech.failures_in_a_row += 1
620
+ if speech.failures_in_a_row >= FAILURES_IN_A_ROW:
621
+ raise JobError(
622
+ f"the voice failed on {FAILURES_IN_A_ROW} sentences in a row, the last "
623
+ f"with: {error}; the sentences spoken so far are kept for a rerun"
624
+ ) from error
625
+ return
626
+ except OSError as error:
627
+ raise JobError(
628
+ f"could not write the voice's audio: {error.strerror or error}; "
629
+ "the sentences spoken so far are kept for a rerun"
630
+ ) from error
631
+ elapsed = speech.clock() - started
632
+ speech.durations[key] = seconds
633
+ speech.spoken += 1
634
+ speech.failures_in_a_row = 0
635
+ speech.characters_done += len(text)
636
+ speech.estimate.done(len(text))
637
+ if speech.first_chapter is None:
638
+ speech.first_chapter = chunk.chapter
639
+ speech.calls += 1
640
+ if speech.calls > WARM_UP_CALLS: # the first calls warm the voice up (D-381)
641
+ speech.estimate.measure(len(text), elapsed)
642
+
643
+
644
+ def _encode(
645
+ work: Path,
646
+ out_folder: Path,
647
+ chapters: list[Chapter],
648
+ spoken: list[Spoken],
649
+ book: BookInfo,
650
+ settings: OutputSettings,
651
+ ffmpeg: _CancellableFfmpeg,
652
+ overwrite: bool,
653
+ ) -> Written:
654
+ """`write_audiobook`, all or nothing for the files that are new: if it fails or is
655
+ cancelled part-way (Ctrl-C included), every audiobook file this run put in place where
656
+ none was before is removed again, so no half set is left and a rerun without
657
+ `overwrite` is not refused. Only files ffmpeg wrote in this run are removed, never a
658
+ file that appeared from elsewhere. With `overwrite`, files this run already replaced
659
+ cannot be restored: the old versions are gone (output removes the old read-along file
660
+ first, so it never vouches for a mixed set, D-119)."""
661
+ _check_rates(work, spoken)
662
+ targets = audiobook_files(out_folder, book, [c.title for c in chapters], settings)
663
+ existed = {target for target in targets if target.exists()}
664
+ try:
665
+ return write_audiobook(
666
+ work, out_folder, chapters, spoken, book, settings, ffmpeg=ffmpeg, overwrite=overwrite
667
+ )
668
+ except BaseException as error:
669
+ _remove_new(targets, existed, ffmpeg.finished)
670
+ if isinstance(error, Exception):
671
+ raise _encode_error(error, out_folder) from error
672
+ raise
673
+
674
+
675
+ def _remove_new(targets: list[Path], existed: set[Path], ours: list[_Identity]) -> None:
676
+ for target in targets:
677
+ if target in existed:
678
+ continue
679
+ with contextlib.suppress(OSError):
680
+ if _identity(target) in ours:
681
+ target.unlink()
682
+
683
+
684
+ def _identity(path: Path) -> _Identity:
685
+ stat = path.stat()
686
+ return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns)
687
+
688
+
689
+ def _encode_error(error: Exception, out_folder: Path) -> Exception:
690
+ """The JobError a person can act on for an error while writing the audiobook."""
691
+ if isinstance(error, JobError):
692
+ return error
693
+ if isinstance(error, FileExistsError):
694
+ return OutputExistsError(f"{error}; choose to replace them")
695
+ if isinstance(error, ChunkAudioError):
696
+ return JobError(f"{error}; run the job again to speak it anew")
697
+ if isinstance(error, FfmpegError | OSError):
698
+ if not out_folder.is_dir():
699
+ return JobError(
700
+ f"the output folder {out_folder.name} was removed during the run; "
701
+ "the spoken audio is kept, run the job again"
702
+ )
703
+ if isinstance(error, FfmpegError):
704
+ return JobError(f"could not write the audiobook: {error}")
705
+ where = Path(str(error.filename)).name if error.filename else "the audiobook"
706
+ return JobError(f"could not write {where}: {error.strerror or error}")
707
+ return error
708
+
709
+
710
+ def _check_rates(work: Path, spoken: list[Spoken]) -> None:
711
+ """All chunks at one sample rate (D-117). They are, unless the cache holds chunks of an
712
+ engine that changed its rate without changing its version; then cleaning helps, and a
713
+ plain rerun would only fail the same way."""
714
+ rates = set()
715
+ for file in dict.fromkeys(s.chunk.file for s in spoken):
716
+ try:
717
+ with wave.open(str(work / file), "rb") as audio:
718
+ rates.add(audio.getframerate())
719
+ except (OSError, EOFError, wave.Error):
720
+ continue # output names a missing or broken chunk itself
721
+ if len(rates) > 1:
722
+ raise JobError(
723
+ f"the working folder holds audio at different sample rates ({sorted(rates)} Hz); "
724
+ "remove its chunks (clean the working folder) and run the job again"
725
+ )
726
+
727
+
728
+ def _skipped(
729
+ result: ScriptResult, plan: AudioPlan, speech: _Speech
730
+ ) -> tuple[SkippedUtterance, ...]:
731
+ pages = Pages(result.document)
732
+ out = []
733
+ for chunk in plan.chunks:
734
+ if chunk.key in speech.failed:
735
+ where = pages.where(chunk.utterance.span)
736
+ out.append(
737
+ SkippedUtterance(
738
+ chapter=chunk.chapter,
739
+ span=chunk.utterance.span,
740
+ page=where.page,
741
+ printed_page=where.printed_page,
742
+ text=chunk.utterance.text,
743
+ error=speech.failed[chunk.key],
744
+ )
745
+ )
746
+ return tuple(out)
747
+
748
+
749
+ def _record(
750
+ result: ScriptResult,
751
+ settings: JobSettings,
752
+ voice: VoiceSource,
753
+ voice_id: str,
754
+ plan: AudioPlan,
755
+ speech: _Speech,
756
+ skipped: tuple[SkippedUtterance, ...],
757
+ written: Written,
758
+ work: Path,
759
+ ) -> dict[str, Any]:
760
+ """`audio.json` (format "sci2audio-audio", version 1, D-165)."""
761
+ source = result.document.content.source
762
+ times = read_readalong(written.readalong)
763
+ seconds = {}
764
+ spoken = [chunk.chapter for chunk in plan.chunks if chunk.key in speech.durations]
765
+ spoken_chapters = list(dict.fromkeys(spoken))
766
+ for index, chapter in zip(spoken_chapters, times.chapters, strict=True):
767
+ seconds[index] = round(chapter.end - chapter.start, 3)
768
+ counts: dict[int, int] = {}
769
+ for chunk in plan.chunks:
770
+ if chunk.key in speech.durations:
771
+ counts[chunk.chapter] = counts.get(chunk.chapter, 0) + 1
772
+ speed = speech.speed()
773
+ return {
774
+ "format": RECORD_FORMAT,
775
+ "version": RECORD_VERSION,
776
+ "sci2audio": __version__,
777
+ "source": {"name": source.name, "format": str(source.format), "sha256": source.sha256},
778
+ "settings": settings_data(settings),
779
+ "engine": voice.engine,
780
+ "engine_version": voice.version,
781
+ "voice": voice_id,
782
+ "speed": settings.speed,
783
+ "duration": times.duration,
784
+ "chapters": [
785
+ {
786
+ "index": chapter.chapter.index,
787
+ "title": chapter.chapter.title,
788
+ "selected": chapter.selected,
789
+ "chunks": counts.get(chapter.chapter.index, 0),
790
+ "seconds": seconds.get(chapter.chapter.index, 0.0),
791
+ }
792
+ for chapter in result.chapters
793
+ ],
794
+ "chunks": {
795
+ "total": len(plan.chunks),
796
+ "spoken": speech.spoken,
797
+ "reused": speech.reused,
798
+ "skipped": len(skipped),
799
+ },
800
+ "skipped": [
801
+ {
802
+ "chapter": item.chapter,
803
+ "block": item.span.block,
804
+ "start": item.span.start,
805
+ "end": item.span.end,
806
+ "page": item.page,
807
+ "printed_page": item.printed_page,
808
+ "text": item.text,
809
+ "error": item.error,
810
+ }
811
+ for item in skipped
812
+ ],
813
+ "characters_per_second": None if speed is None else round(speed, 1),
814
+ "cache_bytes": cache_size(work),
815
+ "files": {
816
+ "m4b": written.m4b.name if written.m4b else None,
817
+ "mp3": [path.name for path in written.mp3s],
818
+ "readalong": written.readalong.name,
819
+ },
820
+ }
821
+
822
+
823
+ def _remove_old_records(work: Path) -> None:
824
+ """The records vouch for the audiobook: an old one must not outlive a rerun that fails
825
+ while it replaces the files (as output does with its read-along file, D-119)."""
826
+ try:
827
+ (work / RECORD).unlink(missing_ok=True)
828
+ (work / DOCUMENT).unlink(missing_ok=True)
829
+ except OSError as error:
830
+ raise JobError(
831
+ f"could not replace the job's record in the working folder: {error.strerror or error}"
832
+ ) from error
833
+
834
+
835
+ def _write_records(work: Path, document: Document, record: dict[str, Any]) -> tuple[Path, Path]:
836
+ """`audio.json` and `document.json` (the laid-out document, for the player, #78)."""
837
+ files = (work / RECORD, work / DOCUMENT)
838
+ try:
839
+ write_texts(
840
+ {files[1]: json_text(to_data(document)), files[0]: json_text(record)},
841
+ overwrite=True,
842
+ )
843
+ except OSError as error:
844
+ raise JobError(
845
+ f"the audiobook is written, but its record could not be: {error.strerror or error}"
846
+ ) from error
847
+ return files