sci2audio 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. sci2audio/__init__.py +3 -0
  2. sci2audio/app/__init__.py +110 -0
  3. sci2audio/app/audio.py +847 -0
  4. sci2audio/app/chapters.py +109 -0
  5. sci2audio/app/chunks.py +102 -0
  6. sci2audio/app/config.py +136 -0
  7. sci2audio/app/estimate.py +192 -0
  8. sci2audio/app/files.py +98 -0
  9. sci2audio/app/job.py +285 -0
  10. sci2audio/app/lock.py +92 -0
  11. sci2audio/app/report.py +400 -0
  12. sci2audio/app/runner.py +92 -0
  13. sci2audio/app/script.py +95 -0
  14. sci2audio/app/settings.py +58 -0
  15. sci2audio/app/voice.py +70 -0
  16. sci2audio/cli.py +1013 -0
  17. sci2audio/core/__init__.py +81 -0
  18. sci2audio/core/audio.py +39 -0
  19. sci2audio/core/content.py +266 -0
  20. sci2audio/core/document.py +225 -0
  21. sci2audio/core/edits.py +121 -0
  22. sci2audio/core/export.py +76 -0
  23. sci2audio/extraction/__init__.py +28 -0
  24. sci2audio/extraction/cover.py +68 -0
  25. sci2audio/extraction/epub.py +576 -0
  26. sci2audio/extraction/errors.py +7 -0
  27. sci2audio/extraction/markdown.py +250 -0
  28. sci2audio/extraction/markup.py +356 -0
  29. sci2audio/extraction/pdf.py +623 -0
  30. sci2audio/extraction/text.py +136 -0
  31. sci2audio/layout/__init__.py +13 -0
  32. sci2audio/layout/build.py +265 -0
  33. sci2audio/layout/captions.py +77 -0
  34. sci2audio/layout/footnotes.py +180 -0
  35. sci2audio/layout/formulas.py +392 -0
  36. sci2audio/layout/furniture.py +326 -0
  37. sci2audio/layout/geometry.py +124 -0
  38. sci2audio/layout/graphics.py +569 -0
  39. sci2audio/layout/headings.py +358 -0
  40. sci2audio/layout/lists.py +167 -0
  41. sci2audio/layout/maths.py +47 -0
  42. sci2audio/layout/order.py +374 -0
  43. sci2audio/layout/paragraphs.py +518 -0
  44. sci2audio/layout/settings.py +142 -0
  45. sci2audio/layout/split.py +113 -0
  46. sci2audio/layout/tagged.py +442 -0
  47. sci2audio/output/__init__.py +44 -0
  48. sci2audio/output/ffmpeg.py +111 -0
  49. sci2audio/output/model.py +125 -0
  50. sci2audio/output/names.py +70 -0
  51. sci2audio/output/readalong.py +160 -0
  52. sci2audio/output/timeline.py +113 -0
  53. sci2audio/output/wav.py +83 -0
  54. sci2audio/output/write.py +280 -0
  55. sci2audio/testing/__init__.py +32 -0
  56. sci2audio/testing/documents.py +124 -0
  57. sci2audio/testing/regression.py +197 -0
  58. sci2audio/testing/text_steps.py +262 -0
  59. sci2audio/testing/voices.py +164 -0
  60. sci2audio/textprep/__init__.py +105 -0
  61. sci2audio/textprep/abbreviations.py +138 -0
  62. sci2audio/textprep/characters.py +267 -0
  63. sci2audio/textprep/citations.py +683 -0
  64. sci2audio/textprep/data/en/abbreviations.toml +209 -0
  65. sci2audio/textprep/data/en/units.toml +548 -0
  66. sci2audio/textprep/footnotes.py +418 -0
  67. sci2audio/textprep/furniture.py +46 -0
  68. sci2audio/textprep/hyphens.py +228 -0
  69. sci2audio/textprep/links.py +628 -0
  70. sci2audio/textprep/notation.py +170 -0
  71. sci2audio/textprep/numbers.py +529 -0
  72. sci2audio/textprep/placeholders.py +68 -0
  73. sci2audio/textprep/references.py +107 -0
  74. sci2audio/textprep/registry.py +83 -0
  75. sci2audio/textprep/runner.py +206 -0
  76. sci2audio/textprep/spoken.py +33 -0
  77. sci2audio/textprep/statistics.py +322 -0
  78. sci2audio/textprep/steps.py +153 -0
  79. sci2audio/textprep/units.py +449 -0
  80. sci2audio/textprep/utterances.py +315 -0
  81. sci2audio/textprep/wording.py +93 -0
  82. sci2audio/textprep/wordlists.py +189 -0
  83. sci2audio/voices/__init__.py +102 -0
  84. sci2audio/voices/download.py +340 -0
  85. sci2audio/voices/engine.py +136 -0
  86. sci2audio/voices/errors.py +88 -0
  87. sci2audio/voices/kokoro_engine.py +346 -0
  88. sci2audio/voices/runtime.py +37 -0
  89. sci2audio/voices/types.py +206 -0
  90. sci2audio/voices/wav.py +82 -0
  91. sci2audio/web/__init__.py +13 -0
  92. sci2audio/web/api.py +658 -0
  93. sci2audio/web/config.py +97 -0
  94. sci2audio/web/documents.py +496 -0
  95. sci2audio/web/i18n.py +188 -0
  96. sci2audio/web/jobs.py +265 -0
  97. sci2audio/web/launch.py +121 -0
  98. sci2audio/web/limits.py +64 -0
  99. sci2audio/web/models.py +178 -0
  100. sci2audio/web/pages.py +509 -0
  101. sci2audio/web/player.py +549 -0
  102. sci2audio/web/security.py +194 -0
  103. sci2audio/web/server.py +116 -0
  104. sci2audio/web/state.py +88 -0
  105. sci2audio/web/static/app.css +150 -0
  106. sci2audio/web/static/app.js +499 -0
  107. sci2audio/web/static/player.js +277 -0
  108. sci2audio/web/strings/de.toml +348 -0
  109. sci2audio/web/strings/en.toml +343 -0
  110. sci2audio/web/tasks.py +280 -0
  111. sci2audio/web/templates/_remove.html +17 -0
  112. sci2audio/web/templates/base.html +32 -0
  113. sci2audio/web/templates/document.html +153 -0
  114. sci2audio/web/templates/index.html +41 -0
  115. sci2audio/web/templates/job.html +66 -0
  116. sci2audio/web/templates/listen.html +75 -0
  117. sci2audio/web/templates/missing.html +7 -0
  118. sci2audio/web/templates/privacy.html +47 -0
  119. sci2audio/web/templates/voice.html +31 -0
  120. sci2audio-1.0.0.dist-info/METADATA +171 -0
  121. sci2audio-1.0.0.dist-info/RECORD +125 -0
  122. sci2audio-1.0.0.dist-info/WHEEL +4 -0
  123. sci2audio-1.0.0.dist-info/entry_points.txt +2 -0
  124. sci2audio-1.0.0.dist-info/licenses/LICENSE +674 -0
  125. sci2audio-1.0.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +86 -0
sci2audio/cli.py ADDED
@@ -0,0 +1,1013 @@
1
+ """The command line (#72, WI-03): `script`, `convert`, `voices` and `clean`.
2
+
3
+ sci2audio script FILE the listening script and the report; no voice needed
4
+ sci2audio convert FILE the audiobook; running it again continues (WI-02)
5
+ sci2audio voices the voices, and whether their files are installed
6
+ sci2audio voices download shows what it fetches, then asks first (VA-03)
7
+ sci2audio clean FILE removes FILE's speech cache
8
+
9
+ Every command is one row of `COMMANDS`: a function that adds its arguments and a function
10
+ that runs it (D-211). `main` builds the real `CliEnv`; tests call `run` with their own. The
11
+ command line reads the settings file as defaults and never writes it (D-213), counts
12
+ chapters from 1 (D-214), prints progress on stderr and the result on stdout (D-220), and
13
+ ends with one of the exit codes in `Exit` (D-212), never with a traceback.
14
+
15
+ Nothing here opens a network connection except `voices download`, after consent, through
16
+ `voices.download_voice_files` (Q12, D-221). `sci2audio.web` and the voice engine are only
17
+ imported when a command needs them, so `sci2audio --version` starts at once.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import argparse
23
+ import contextlib
24
+ import sys
25
+ import threading
26
+ from collections.abc import Callable, Sequence
27
+ from dataclasses import dataclass, field, replace
28
+ from enum import IntEnum
29
+ from pathlib import Path
30
+ from typing import IO, Any, TextIO
31
+ from urllib.parse import urlsplit
32
+
33
+ from sci2audio import __version__
34
+ from sci2audio.app import (
35
+ AudioJob,
36
+ AudioProgress,
37
+ JobBusyError,
38
+ JobCancelledError,
39
+ JobError,
40
+ JobRunner,
41
+ JobSettings,
42
+ OutputExistsError,
43
+ Phase,
44
+ ScriptJob,
45
+ ScriptResult,
46
+ UnreadableFileError,
47
+ UnsupportedFileError,
48
+ VoiceSource,
49
+ clean_work_folder,
50
+ default_config_file,
51
+ default_models_folder,
52
+ default_work_folder,
53
+ encoding_warning,
54
+ find_chapters,
55
+ kokoro_voice,
56
+ load_settings,
57
+ script_files,
58
+ )
59
+ from sci2audio.app.chunks import CHUNKS, cache_size
60
+ from sci2audio.app.job import read_document
61
+ from sci2audio.app.settings import MAX_SPEED, MIN_SPEED
62
+ from sci2audio.output import OutputSettings
63
+ from sci2audio.textprep import Category
64
+ from sci2audio.voices import (
65
+ DEFAULT_VOICE,
66
+ ENGLISH_VOICES,
67
+ DownloadError,
68
+ DownloadPlan,
69
+ NetworkError,
70
+ RemoteFile,
71
+ download_plan,
72
+ download_voice_files,
73
+ missing_files,
74
+ )
75
+ from sci2audio.voices.download import Fetch
76
+
77
+ PROG = "sci2audio"
78
+
79
+
80
+ class Exit(IntEnum):
81
+ """The exit codes (D-212); the README lists them."""
82
+
83
+ OK = 0
84
+ FAILED = 1 # any other failure: a write, the disk, ffmpeg, the voice, the download
85
+ USAGE = 2 # a bad option or value (argparse uses 2 too)
86
+ NOT_FOUND = 3 # FILE does not exist or is not a file
87
+ UNSUPPORTED = 4 # a file type sci2audio can't read
88
+ UNREADABLE = 5 # damaged, password-protected or can't be opened
89
+ NO_VOICE = 6 # the voice files are not installed
90
+ EXISTS = 7 # output files exist and --overwrite was not given
91
+ BUSY = 8 # another sci2audio job is working on this file
92
+ INTERRUPTED = 130 # Ctrl-C (128 + SIGINT)
93
+
94
+
95
+ EXIT_CODES_HELP = """\
96
+ exit codes:
97
+ 0 done (problems listed in the report do not change it)
98
+ 1 failed for another reason (a write, the disk, ffmpeg, the voice, the download)
99
+ 2 a bad option or value
100
+ 3 the file does not exist or is not a file
101
+ 4 a file type sci2audio can't read
102
+ 5 the file is damaged, protected or can't be opened
103
+ 6 the voice files are not installed (run: sci2audio voices download)
104
+ 7 output files exist (add --overwrite to replace them)
105
+ 8 another sci2audio job is working on this file
106
+ 130 stopped with Ctrl-C
107
+ """
108
+
109
+ # Where the two download hosts send the files (D-141); shown before consent.
110
+ REDIRECT_HOSTS = {
111
+ "huggingface.co": "its CDN and its LFS/Xet storage",
112
+ "github.com": "objects.githubusercontent.com",
113
+ }
114
+
115
+
116
+ class CliError(Exception):
117
+ """Stops a command with a message and an exit code."""
118
+
119
+ def __init__(self, message: str, code: Exit) -> None:
120
+ super().__init__(message)
121
+ self.code = code
122
+
123
+
124
+ @dataclass
125
+ class CliEnv:
126
+ """Everything a command touches outside its arguments, so tests can replace it."""
127
+
128
+ out: TextIO
129
+ err: TextIO
130
+ ask: Callable[[str], str] # asks a question on the terminal; EOFError without one
131
+ interactive: bool # a person can answer questions (stdin is a terminal)
132
+ live: bool # stderr is a terminal: progress is rewritten in one line
133
+ runner: JobRunner = field(default_factory=JobRunner)
134
+ config_file: Callable[[], Path] = default_config_file
135
+ models_folder: Callable[[], Path] = default_models_folder
136
+ work_folder: Callable[[Path], Path] = default_work_folder
137
+ voice_source: Callable[[Path], VoiceSource] = kokoro_voice
138
+ missing_voice_files: Callable[[Path], list[RemoteFile]] = missing_files
139
+ fetch: Fetch | None = None # the download's fetch; None = the real one
140
+
141
+ @classmethod
142
+ def real(cls) -> CliEnv:
143
+ for stream in (sys.stdout, sys.stderr):
144
+ _never_fail_on_characters(stream)
145
+ return cls(
146
+ out=sys.stdout,
147
+ err=sys.stderr,
148
+ ask=_ask,
149
+ interactive=_is_terminal(sys.stdin),
150
+ live=_is_terminal(sys.stderr),
151
+ )
152
+
153
+
154
+ def _ask(question: str) -> str:
155
+ """Ask on stderr and read the answer from stdin; EOFError at the end of input."""
156
+ sys.stderr.write(question)
157
+ sys.stderr.flush()
158
+ line = sys.stdin.readline()
159
+ if not line:
160
+ raise EOFError
161
+ return line
162
+
163
+
164
+ def _never_fail_on_characters(stream: IO[str]) -> None:
165
+ """A console that can't show a character prints "?" instead of failing (D-225).
166
+
167
+ A redirected stdout on Windows uses the locale's code page (cp1252), which lacks many
168
+ letters of names and maths; the encoding stays as it is, so pipes and files keep the
169
+ system's choice."""
170
+ reconfigure = getattr(stream, "reconfigure", None)
171
+ if reconfigure is not None:
172
+ with contextlib.suppress(ValueError, OSError):
173
+ reconfigure(errors="replace")
174
+
175
+
176
+ def _is_terminal(stream: IO[str] | None) -> bool:
177
+ try:
178
+ return stream is not None and stream.isatty()
179
+ except (ValueError, OSError):
180
+ return False
181
+
182
+
183
+ # ---- parsing --------------------------------------------------------------------------
184
+
185
+
186
+ class _Formatter(argparse.HelpFormatter):
187
+ """Wraps one-paragraph texts; keeps texts with line breaks (the epilogs) as written."""
188
+
189
+ def _fill_text(self, text: str, width: int, indent: str) -> str:
190
+ if "\n" in text.strip():
191
+ return text
192
+ return super()._fill_text(text, width, indent)
193
+
194
+
195
+ class _Parser(argparse.ArgumentParser):
196
+ """argparse, writing to the env's streams instead of sys.stdout and sys.stderr."""
197
+
198
+ def __init__(self, *args: Any, env: CliEnv, **kwargs: Any) -> None:
199
+ kwargs.setdefault("formatter_class", _Formatter)
200
+ super().__init__(*args, **kwargs)
201
+ self.env = env
202
+
203
+ def _print_message(self, message: str, file: IO[str] | None = None) -> None:
204
+ if not message:
205
+ return
206
+ if file is None or file is sys.stdout:
207
+ file = self.env.out
208
+ elif file is sys.stderr:
209
+ file = self.env.err
210
+ file.write(message)
211
+
212
+ def subcommands(self, **kwargs: Any) -> argparse._SubParsersAction[_Parser]:
213
+ return self.add_subparsers(parser_class=_parser_class(self.env), **kwargs)
214
+
215
+
216
+ def _parser_class(env: CliEnv) -> Callable[..., _Parser]:
217
+ def make(**kwargs: Any) -> _Parser:
218
+ return _Parser(env=env, **kwargs)
219
+
220
+ return make
221
+
222
+
223
+ def chapter_numbers(text: str) -> frozenset[int] | None:
224
+ """`--chapters`: "all" (None here, resolved once the chapters are known) or numbers
225
+ and ranges counted from 1 ("1,3-5"), as `JobSettings.chapters` indices from 0."""
226
+ if text.strip().lower() == "all":
227
+ return None
228
+ numbers: set[int] = set()
229
+ for part in text.split(","):
230
+ part = part.strip()
231
+ first, dash, last = part.partition("-")
232
+ try:
233
+ low = int(first)
234
+ high = int(last) if dash else low
235
+ except ValueError:
236
+ raise argparse.ArgumentTypeError(
237
+ f"{part!r} is not a chapter number; use numbers and ranges like 1,3-5, or all"
238
+ ) from None
239
+ if low < 1 or high < low:
240
+ raise argparse.ArgumentTypeError(
241
+ f"{part!r}: chapters are counted from 1, and a range goes upwards"
242
+ )
243
+ if high - low > 10_000:
244
+ raise argparse.ArgumentTypeError(f"{part!r}: that range is too long")
245
+ numbers.update(range(low - 1, high))
246
+ return frozenset(numbers)
247
+
248
+
249
+ def categories(text: str) -> frozenset[Category]:
250
+ """`--on` / `--off`: one or more category names, comma-separated."""
251
+ found = set()
252
+ for name in text.split(","):
253
+ name = name.strip().lower()
254
+ try:
255
+ found.add(Category(name))
256
+ except ValueError:
257
+ known = ", ".join(category.value for category in Category)
258
+ raise argparse.ArgumentTypeError(
259
+ f"unknown category {name!r}; the categories are: {known}"
260
+ ) from None
261
+ return frozenset(found)
262
+
263
+
264
+ def speed(text: str) -> float:
265
+ try:
266
+ value = float(text)
267
+ except ValueError:
268
+ value = float("nan")
269
+ if not MIN_SPEED <= value <= MAX_SPEED: # also false for nan
270
+ raise argparse.ArgumentTypeError(
271
+ f"the speed must be a number from {MIN_SPEED} to {MAX_SPEED}, got {text!r}"
272
+ )
273
+ return value
274
+
275
+
276
+ def _categories_help() -> str:
277
+ lines = ["text-step categories (for --on and --off):"]
278
+ for category in Category:
279
+ start = "on" if category.on_by_default else "off"
280
+ lines.append(f" {category.value:<20} {category.label} (starts {start})")
281
+ lines.append(
282
+ ' "on" means sci2audio handles that kind of text (e.g. citations are left out);\n'
283
+ ' "off" means it is read as printed.'
284
+ )
285
+ return "\n".join(lines)
286
+
287
+
288
+ def _add_job_options(parser: _Parser) -> None:
289
+ """The options `script` and `convert` share."""
290
+ parser.add_argument("file", type=Path, metavar="FILE", help="the file to read")
291
+ parser.add_argument(
292
+ "-o",
293
+ "--output",
294
+ type=Path,
295
+ metavar="FOLDER",
296
+ help="where the files go (default: next to FILE)",
297
+ )
298
+ parser.add_argument(
299
+ "--chapters",
300
+ type=chapter_numbers,
301
+ metavar="LIST",
302
+ default=argparse.SUPPRESS,
303
+ help="chapters to read, counted from 1 as `script` lists them: 1,3-5 or all "
304
+ "(default: every chapter but the reference list)",
305
+ )
306
+ parser.add_argument(
307
+ "--on",
308
+ type=categories,
309
+ action="append",
310
+ default=[],
311
+ metavar="CATEGORY",
312
+ help="switch a text-step category on (repeat, or separate with commas)",
313
+ )
314
+ parser.add_argument(
315
+ "--off",
316
+ type=categories,
317
+ action="append",
318
+ default=[],
319
+ metavar="CATEGORY",
320
+ help="switch a text-step category off: its text is read as printed",
321
+ )
322
+ parser.add_argument(
323
+ "--language", metavar="TAG", help="en, en-GB, ... (default: the file's own)"
324
+ )
325
+ parser.add_argument("--overwrite", action="store_true", help="replace existing output files")
326
+ parser.add_argument(
327
+ "--work-folder",
328
+ type=Path,
329
+ metavar="FOLDER",
330
+ help="the job's working folder (default: in the cache folder)",
331
+ )
332
+ parser.add_argument(
333
+ "--ignore-settings",
334
+ action="store_true",
335
+ help="use the built-in defaults, not the settings saved by the interface",
336
+ )
337
+ parser.add_argument("-q", "--quiet", action="store_true", help="no progress lines")
338
+
339
+
340
+ def _add_models_option(parser: _Parser, *, default: Any = None) -> None:
341
+ parser.add_argument(
342
+ "--models-folder",
343
+ type=Path,
344
+ metavar="FOLDER",
345
+ default=default,
346
+ help="where the voice files are (default: in the user's data folder)",
347
+ )
348
+
349
+
350
+ def add_script(parser: _Parser) -> None:
351
+ _add_job_options(parser)
352
+
353
+
354
+ def add_convert(parser: _Parser) -> None:
355
+ _add_job_options(parser)
356
+ parser.add_argument(
357
+ "--voice", metavar="ID", help=f"a voice from `{PROG} voices` (default: {DEFAULT_VOICE})"
358
+ )
359
+ parser.add_argument(
360
+ "--speed", type=speed, metavar="X", help=f"{MIN_SPEED} to {MAX_SPEED} (default: 1.0)"
361
+ )
362
+ parser.add_argument(
363
+ "--format",
364
+ choices=("both", "m4b", "mp3"),
365
+ default="both",
366
+ help="M4B with chapter marks, one MP3 per chapter, or both (default: both)",
367
+ )
368
+ parser.add_argument(
369
+ "--with-script",
370
+ action="store_true",
371
+ help="also write the listening script and the report",
372
+ )
373
+ _add_models_option(parser)
374
+
375
+
376
+ def add_voices(parser: _Parser) -> None:
377
+ _add_models_option(parser)
378
+ actions = parser.subcommands(dest="action", metavar="ACTION")
379
+ download = actions.add_parser(
380
+ "download",
381
+ help="download the voice files (about 355 MB), after showing what and asking",
382
+ description="Download the voice files: the Kokoro model with its English voices and "
383
+ "the spaCy English model. Shows every source, size and licence first, then asks.",
384
+ )
385
+ _add_models_option(download, default=argparse.SUPPRESS) # keeps `voices --models-folder`
386
+ download.add_argument(
387
+ "-y", "--yes", action="store_true", help="agree to the download without being asked"
388
+ )
389
+
390
+
391
+ def add_clean(parser: _Parser) -> None:
392
+ parser.add_argument("file", type=Path, metavar="FILE", help="the file whose speech cache goes")
393
+ parser.add_argument(
394
+ "--work-folder",
395
+ type=Path,
396
+ metavar="FOLDER",
397
+ help="the job's working folder (default: in the cache folder)",
398
+ )
399
+
400
+
401
+ def add_ui(parser: _Parser) -> None:
402
+ parser.add_argument(
403
+ "--no-browser", action="store_true", help="only print the link; do not open a browser"
404
+ )
405
+
406
+
407
+ @dataclass(frozen=True)
408
+ class Command:
409
+ name: str
410
+ help: str
411
+ description: str
412
+ add_arguments: Callable[[_Parser], None]
413
+ run: Callable[[argparse.Namespace, CliEnv], Exit]
414
+ epilog: Callable[[], str] | None = None
415
+
416
+
417
+ def build_parser(env: CliEnv) -> _Parser:
418
+ parser = _Parser(
419
+ prog=PROG,
420
+ env=env,
421
+ description="Turn scientific texts into faithful, offline audiobooks.",
422
+ epilog=EXIT_CODES_HELP,
423
+ )
424
+ parser.add_argument("--version", action="version", version=f"{PROG} {__version__}")
425
+ commands = parser.subcommands(dest="command", metavar="COMMAND")
426
+ for command in COMMANDS:
427
+ sub = commands.add_parser(
428
+ command.name,
429
+ help=command.help,
430
+ description=command.description,
431
+ epilog=command.epilog() if command.epilog else None,
432
+ )
433
+ command.add_arguments(sub)
434
+ sub.set_defaults(run=command.run)
435
+ return parser
436
+
437
+
438
+ # ---- the shared job set-up --------------------------------------------------------------
439
+
440
+
441
+ def _source(file: Path) -> Path:
442
+ if not file.exists():
443
+ raise CliError(f"{file}: no such file", Exit.NOT_FOUND)
444
+ if not file.is_file():
445
+ raise CliError(f"{file}: not a file", Exit.NOT_FOUND)
446
+ return file
447
+
448
+
449
+ def _work_folder(args: argparse.Namespace, env: CliEnv, source: Path) -> Path:
450
+ if args.work_folder is not None:
451
+ return args.work_folder
452
+ try:
453
+ return env.work_folder(source)
454
+ except OSError as error:
455
+ raise CliError(
456
+ f"{source}: cannot be read ({error.strerror or error})", Exit.UNREADABLE
457
+ ) from error
458
+
459
+
460
+ def _settings(args: argparse.Namespace, env: CliEnv) -> JobSettings:
461
+ """The settings file's choices (D-213), with this run's options on top. `--chapters
462
+ all` is left empty here: the caller resolves it with `_all_chapters` once every check
463
+ that needs no reading is done (D-222)."""
464
+ base = JobSettings()
465
+ if not args.ignore_settings:
466
+ base, warnings = load_settings(env.config_file())
467
+ for warning in warnings:
468
+ env.err.write(f"{PROG}: warning: {warning}\n")
469
+ on = frozenset().union(*args.on)
470
+ off = frozenset().union(*args.off)
471
+ if both := on & off:
472
+ names = ", ".join(sorted(category.value for category in both))
473
+ raise CliError(f"{names}: switched both on and off; choose one", Exit.USAGE)
474
+ changes: dict[str, Any] = {"switched_off": (base.switched_off - on) | off}
475
+ if args.output is not None:
476
+ changes["output_folder"] = args.output
477
+ if args.language is not None:
478
+ changes["language"] = args.language
479
+ if hasattr(args, "chapters"):
480
+ changes["chapters"] = args.chapters if args.chapters is not None else frozenset()
481
+ for name in ("voice", "speed"):
482
+ if getattr(args, name, None) is not None:
483
+ changes[name] = getattr(args, name)
484
+ try:
485
+ settings = replace(base, **changes) # checks every value again
486
+ except ValueError as error:
487
+ raise CliError(str(error), Exit.USAGE) from error
488
+ return settings
489
+
490
+
491
+ def _every_chapter_asked(args: argparse.Namespace) -> bool:
492
+ return hasattr(args, "chapters") and args.chapters is None # --chapters all
493
+
494
+
495
+ def _all_chapters(source: Path, settings: JobSettings) -> JobSettings:
496
+ """`--chapters all`: the chapter count is only known once the file is read. (The job
497
+ reads it again; Issue #112 lets `JobSettings` say "every chapter" instead.)"""
498
+ document = read_document(source, settings, lambda phase: None)
499
+ chapters = frozenset(chapter.index for chapter in find_chapters(document))
500
+ return replace(settings, chapters=chapters)
501
+
502
+
503
+ def _guessed_encoding(env: CliEnv, result: ScriptResult) -> None:
504
+ """Say when a text file's encoding was a guess (D-346)."""
505
+ if warning := encoding_warning(result.document.content.source):
506
+ env.err.write(f"{PROG}: warning: {warning}\n")
507
+
508
+
509
+ def _ignored(env: CliEnv, ignored: Sequence[int], count: int) -> None:
510
+ for index in ignored:
511
+ env.err.write(
512
+ f"{PROG}: warning: there is no chapter {index + 1} (the document has chapters "
513
+ f"1 to {count}); ignored\n"
514
+ )
515
+
516
+
517
+ # ---- progress ---------------------------------------------------------------------------
518
+
519
+ PHASE_TEXT = {
520
+ Phase.READING: "Reading the file",
521
+ Phase.LAYOUT: "Finding the reading order",
522
+ Phase.TEXT_STEPS: "Preparing the text",
523
+ Phase.LOADING_VOICE: "Loading the voice",
524
+ Phase.SPEAKING: "Speaking",
525
+ Phase.ENCODING: "Writing the audio files",
526
+ Phase.WRITING: "Writing the script and the report",
527
+ }
528
+
529
+
530
+ class Progress:
531
+ """Progress lines on stderr (D-220): one line rewritten in place on a terminal; one
532
+ line per phase and per chapter otherwise; nothing with --quiet. Thread-safe."""
533
+
534
+ def __init__(self, env: CliEnv, quiet: bool) -> None:
535
+ self.env = env
536
+ self.quiet = quiet
537
+ self.width = 0 # of the line shown in place
538
+ self.last = "" # the last line written without a terminal
539
+ self.lock = threading.Lock()
540
+
541
+ def show(self, text: str, *, key: str | None = None) -> None:
542
+ """Show `text`; without a terminal only when `key` (default: the text) changed."""
543
+ if self.quiet:
544
+ return
545
+ with self.lock:
546
+ if self.env.live:
547
+ pad = " " * max(0, self.width - len(text))
548
+ self.env.err.write(f"\r{text}{pad}")
549
+ self.width = len(text)
550
+ else:
551
+ key = text if key is None else key
552
+ if key != self.last:
553
+ self.env.err.write(text + "\n")
554
+ self.last = key
555
+ self.env.err.flush()
556
+
557
+ def end(self) -> None:
558
+ """Finish the line shown in place, before other output."""
559
+ with self.lock:
560
+ if self.width:
561
+ self.env.err.write("\n")
562
+ self.env.err.flush()
563
+ self.width = 0
564
+
565
+ def phase(self, phase: Phase) -> None:
566
+ if phase in PHASE_TEXT:
567
+ self.show(PHASE_TEXT[phase] + " ...")
568
+
569
+ def audio(self, event: AudioProgress) -> None:
570
+ if event.phase is not Phase.SPEAKING or event.chapter is None:
571
+ self.phase(event.phase)
572
+ return
573
+ left = _minutes(event.seconds_left)
574
+ text = (
575
+ f"Speaking chapter {event.chapter + 1}, {_short(event.chapter_title)}: "
576
+ f"{event.chunks_done} of {event.chunks_total} sentences, {left}"
577
+ )
578
+ self.show(text, key=f"chapter {event.chapter}")
579
+
580
+ def download(self, done: int, total: int, target: str) -> None:
581
+ name = target.rsplit("/", 1)[-1]
582
+ tenth = done * 10 // total if total else 10
583
+ self.show(f"Downloading: {_mb(done)} of {_mb(total)} ({name})", key=f"{name} {tenth}")
584
+
585
+
586
+ def _minutes(seconds: float | None) -> str:
587
+ if seconds is None:
588
+ return "measuring the speed"
589
+ if seconds < 90:
590
+ return "about a minute left"
591
+ if seconds < 90 * 60:
592
+ return f"about {round(seconds / 60)} min left"
593
+ return f"about {seconds / 3600:.1f} hours left"
594
+
595
+
596
+ def _short(title: str, limit: int = 40) -> str:
597
+ title = " ".join(title.split())
598
+ return title if len(title) <= limit else title[: limit - 3] + "..."
599
+
600
+
601
+ def _mb(size: int) -> str:
602
+ return f"{size / 1_000_000:,.1f} MB" if size < 10_000_000 else f"{size / 1_000_000:,.0f} MB"
603
+
604
+
605
+ def _duration(seconds: float) -> str:
606
+ whole = round(seconds)
607
+ hours, rest = divmod(whole, 3600)
608
+ return f"{hours}:{rest // 60:02d}:{rest % 60:02d}"
609
+
610
+
611
+ # ---- script -----------------------------------------------------------------------------
612
+
613
+
614
+ def run_script_command(args: argparse.Namespace, env: CliEnv) -> Exit:
615
+ source = _source(args.file)
616
+ settings = _settings(args, env)
617
+ if _every_chapter_asked(args):
618
+ settings = _all_chapters(source, settings)
619
+ work = _work_folder(args, env, source)
620
+ progress = Progress(env, args.quiet)
621
+ try:
622
+ job = env.runner.run_script(
623
+ source, settings, work, overwrite=args.overwrite, progress=progress.phase
624
+ )
625
+ finally:
626
+ progress.end()
627
+ _print_script(env, job)
628
+ return Exit.OK
629
+
630
+
631
+ def _print_script(env: CliEnv, job: ScriptJob) -> None:
632
+ out = env.out
633
+ report = job.report
634
+ _guessed_encoding(env, job.result)
635
+ _ignored(env, job.result.ignored_chapters, len(report.chapters))
636
+ out.write("Chapters (choose with --chapters):\n")
637
+ for entry in report.chapters:
638
+ mark = "x" if entry.selected else " "
639
+ note = " (reference list)" if entry.chapter.references else ""
640
+ said = f"{entry.utterances} sentences" if entry.selected else "not read"
641
+ title = _short(entry.chapter.title, 60)
642
+ out.write(f" {entry.chapter.index + 1:>3} [{mark}] {title} {said}{note}\n")
643
+ applied = [(count.category.label, count.applied) for count in report.counts if count.applied]
644
+ if applied:
645
+ out.write("Edits that change what is heard:\n")
646
+ for label, number in applied:
647
+ out.write(f" {label}: {number}\n")
648
+ for step, message in report.problems:
649
+ env.err.write(f"{PROG}: warning: the text step {step} failed and was left out: {message}\n")
650
+ out.write("Wrote:\n")
651
+ for path in job.files.outputs:
652
+ out.write(f" {path}\n")
653
+
654
+
655
+ # ---- convert ----------------------------------------------------------------------------
656
+
657
+
658
+ def run_convert_command(args: argparse.Namespace, env: CliEnv) -> Exit:
659
+ source = _source(args.file)
660
+ models = args.models_folder or env.models_folder()
661
+ if env.missing_voice_files(models):
662
+ raise CliError(
663
+ f"the voice files are not installed in {models}; run `{PROG} voices download` first",
664
+ Exit.NO_VOICE,
665
+ )
666
+ voice = env.voice_source(models)
667
+ settings = _settings(args, env)
668
+ voice_id = settings.voice or voice.default_voice
669
+ if voice_id not in voice.voices:
670
+ raise CliError(
671
+ f"there is no voice {voice_id!r}; `{PROG} voices` lists the voices", Exit.USAGE
672
+ )
673
+ if _every_chapter_asked(args):
674
+ settings = _all_chapters(source, settings)
675
+ work = _work_folder(args, env, source)
676
+ output = OutputSettings(mp3=args.format != "m4b", m4b=args.format != "mp3")
677
+ if args.with_script and not args.overwrite:
678
+ # all or nothing: refuse an existing script before any work, as the audio job
679
+ # refuses an existing audiobook before any speech (D-163); the audio job runs
680
+ # first, so a refused audiobook leaves no new script behind
681
+ targets = script_files(source, settings, work).outputs
682
+ if existing := [path.name for path in targets if path.exists()]:
683
+ raise OutputExistsError(
684
+ f"these files exist already: {', '.join(existing)}; choose to replace them"
685
+ )
686
+ progress = Progress(env, args.quiet)
687
+ script: ScriptJob | None = None
688
+ try:
689
+ job = _audio_in_thread(env, progress, source, settings, work, voice, output, args.overwrite)
690
+ if args.with_script:
691
+ script = env.runner.run_script(
692
+ source, settings, work, overwrite=args.overwrite, progress=progress.phase
693
+ )
694
+ finally:
695
+ progress.end()
696
+ _print_audio(env, job, script, work)
697
+ return Exit.OK
698
+
699
+
700
+ def _audio_in_thread(
701
+ env: CliEnv,
702
+ progress: Progress,
703
+ source: Path,
704
+ settings: JobSettings,
705
+ work: Path,
706
+ voice: VoiceSource,
707
+ output: OutputSettings,
708
+ overwrite: bool,
709
+ ) -> AudioJob:
710
+ """Run the audio job in a worker thread, so the first Ctrl-C can ask it to stop at its
711
+ next cancel point (D-219); a second Ctrl-C stops at once."""
712
+ cancel = threading.Event()
713
+ finished = threading.Event()
714
+ outcome: dict[str, Any] = {}
715
+
716
+ def work_on() -> None:
717
+ try:
718
+ outcome["job"] = env.runner.run_audio(
719
+ source,
720
+ settings,
721
+ work,
722
+ voice=voice,
723
+ output_settings=output,
724
+ overwrite=overwrite,
725
+ progress=progress.audio,
726
+ cancel=cancel,
727
+ )
728
+ except BaseException as error: # handed to the main thread
729
+ outcome["error"] = error
730
+ finally:
731
+ finished.set()
732
+
733
+ worker = threading.Thread(target=work_on, name="sci2audio-audio-job", daemon=True)
734
+ try:
735
+ worker.start() # inside the try: a Ctrl-C while it starts is a first Ctrl-C too
736
+ _wait(finished)
737
+ except KeyboardInterrupt:
738
+ if worker.ident is None: # it never started: nothing to stop politely
739
+ raise
740
+ cancel.set()
741
+ progress.end()
742
+ env.err.write(f"{PROG}: stopping after the current sentence; finished sentences are kept\n")
743
+ env.err.flush()
744
+ _wait(finished) # a second Ctrl-C ends the program here
745
+ if "error" in outcome:
746
+ raise outcome["error"]
747
+ job: AudioJob = outcome["job"]
748
+ return job
749
+
750
+
751
+ def _wait(finished: threading.Event) -> None:
752
+ """Wait in short steps, so Ctrl-C is seen at once on every system. (Thread.join can't
753
+ be used: interrupted, it may report a running thread as finished before Python 3.13.)"""
754
+ while not finished.wait(0.1):
755
+ pass
756
+
757
+
758
+ def _print_audio(env: CliEnv, job: AudioJob, script: ScriptJob | None, work: Path) -> None:
759
+ out = env.out
760
+ _guessed_encoding(env, job.result)
761
+ _ignored(env, job.result.ignored_chapters, len(job.result.chapters))
762
+ total = job.spoken + job.reused
763
+ out.write(
764
+ f"Spoke {job.spoken} of {total} sentences, reused {job.reused} from the last run; "
765
+ f"the book is {_duration(job.duration)} long.\n"
766
+ )
767
+ for item in job.skipped:
768
+ page = f" on page {item.printed_page or item.page}" if item.page else ""
769
+ env.err.write(
770
+ f"{PROG}: warning: the voice could not say this sentence{page}, so it is left out: "
771
+ f"{_short(item.text, 60)!r} ({item.error})\n"
772
+ )
773
+ out.write("Wrote:\n")
774
+ files = [
775
+ *([job.written.m4b] if job.written.m4b else []),
776
+ *job.written.mp3s,
777
+ job.written.readalong,
778
+ ]
779
+ if script is not None:
780
+ files += list(script.files.outputs)
781
+ for path in files:
782
+ out.write(f" {path}\n")
783
+ if cache := cache_size(work):
784
+ out.write(
785
+ f"The speech cache in {work} holds {_mb(cache)}; it makes a rerun quick. "
786
+ f"`{PROG} clean {_quoted(job.result.document.content.source.name)}` removes it.\n"
787
+ )
788
+
789
+
790
+ def _quoted(name: str) -> str:
791
+ return f'"{name}"' if any(c.isspace() or c in "'\"$&|;<>()" for c in name) else name
792
+
793
+
794
+ # ---- voices -----------------------------------------------------------------------------
795
+
796
+
797
+ def run_voices_command(args: argparse.Namespace, env: CliEnv) -> Exit:
798
+ models = args.models_folder or env.models_folder()
799
+ if args.action == "download":
800
+ return _download(args, env, models)
801
+ out = env.out
802
+ out.write("Voices (Kokoro-82M, English):\n")
803
+ out.write(f" {'ID':<12} {'NAME':<10} {'ACCENT':<7} GENDER\n")
804
+ for voice in ENGLISH_VOICES:
805
+ default = " (default)" if voice.id == DEFAULT_VOICE else ""
806
+ out.write(
807
+ f" {voice.id:<12} {voice.name:<10} {voice.language:<7} {voice.gender}{default}\n"
808
+ )
809
+ missing = env.missing_voice_files(models)
810
+ if missing:
811
+ size = sum(f.size for f in missing)
812
+ out.write(
813
+ f"The voice files are not installed in {models} ({_mb(size)} to download).\n"
814
+ f"Run `{PROG} voices download` to see what is fetched and to agree to it.\n"
815
+ )
816
+ else:
817
+ out.write(f"The voice files are installed in {models}.\n")
818
+ return Exit.OK
819
+
820
+
821
+ def describe_download(plan: DownloadPlan, missing: Sequence[RemoteFile], models: Path) -> str:
822
+ """What the download fetches, from where and under which licence (VA-03, D-221)."""
823
+ lines = [f"The voice download puts these files into {models}:"]
824
+ groups: dict[str, list[RemoteFile]] = {}
825
+ for file in plan.files:
826
+ groups.setdefault(file.licence, []).append(file)
827
+ for licence, files in groups.items():
828
+ hosts = sorted({urlsplit(f.url).hostname or "?" for f in files})
829
+ size = sum(f.size for f in files)
830
+ noun = "file" if len(files) == 1 else "files"
831
+ lines.append(f"- {len(files)} {noun}, {_mb(size)}, from {', '.join(hosts)}")
832
+ lines.append(f" {licence}")
833
+ lines.append(f"In total {_mb(plan.total_bytes)}.")
834
+ hosts = sorted({urlsplit(f.url).hostname or "" for f in plan.files} & REDIRECT_HOSTS.keys())
835
+ if hosts:
836
+ served = "; ".join(f"{host} from {REDIRECT_HOSTS[host]}" for host in hosts)
837
+ lines.append(f"The hosts hand the files out from their download servers: {served}.")
838
+ lines.append(
839
+ "Every file is checked against its SHA-256 checksum. Nothing about you or your "
840
+ "files is sent."
841
+ )
842
+ if len(missing) < len(plan.files):
843
+ lines.append(
844
+ f"{len(plan.files) - len(missing)} files are in place already; "
845
+ f"{_mb(sum(f.size for f in missing))} remain."
846
+ )
847
+ return "\n".join(lines) + "\n"
848
+
849
+
850
+ def _download(args: argparse.Namespace, env: CliEnv, models: Path) -> Exit:
851
+ plan = download_plan()
852
+ missing = env.missing_voice_files(models)
853
+ if not missing:
854
+ env.out.write(f"The voice files are installed in {models}; nothing to download.\n")
855
+ return Exit.OK
856
+ # the plan and the question go to stderr, beside the progress: with stdout redirected
857
+ # to a file the person still sees what they are asked to agree to
858
+ env.err.write(describe_download(plan, missing, models))
859
+ env.err.flush()
860
+ if not args.yes:
861
+ if not env.interactive:
862
+ raise CliError(
863
+ "the voice download needs your agreement: run it in a terminal, "
864
+ "or add --yes to agree on the command line",
865
+ Exit.USAGE,
866
+ )
867
+ size = _mb(sum(f.size for f in missing))
868
+ try:
869
+ answer = env.ask(f"Download about {size} now? [y/N] ")
870
+ except EOFError:
871
+ answer = ""
872
+ if answer.strip().lower() not in ("y", "yes"):
873
+ env.out.write("Nothing was downloaded.\n")
874
+ return Exit.FAILED
875
+ progress = Progress(env, quiet=False)
876
+ try:
877
+ download_voice_files(models, progress=progress.download, fetch=env.fetch)
878
+ except (NetworkError, DownloadError) as error:
879
+ progress.end()
880
+ raise CliError(
881
+ f"{error}; the files finished so far are kept, so running the download again continues",
882
+ Exit.FAILED,
883
+ ) from error
884
+ finally:
885
+ progress.end()
886
+ env.out.write(f"The voice files are installed in {models}.\n")
887
+ return Exit.OK
888
+
889
+
890
+ # ---- clean ------------------------------------------------------------------------------
891
+
892
+
893
+ def run_clean_command(args: argparse.Namespace, env: CliEnv) -> Exit:
894
+ source = _source(args.file)
895
+ work = _work_folder(args, env, source)
896
+ if not (work / CHUNKS).exists():
897
+ env.out.write(f"{source.name} has no speech cache; nothing to remove.\n")
898
+ return Exit.OK
899
+ freed = clean_work_folder(work)
900
+ env.out.write(f"Removed the speech cache of {source.name} ({_mb(freed)}).\n")
901
+ return Exit.OK
902
+
903
+
904
+ # ---- ui ---------------------------------------------------------------------------------
905
+
906
+
907
+ def run_ui_command(args: argparse.Namespace, env: CliEnv) -> Exit:
908
+ from sci2audio.web import run as serve # loads the web framework only for this command
909
+
910
+ return Exit(serve(open_browser=not args.no_browser))
911
+
912
+
913
+ COMMANDS: tuple[Command, ...] = (
914
+ Command(
915
+ "script",
916
+ "write the listening script and the report (no voice needed)",
917
+ "Read FILE and write what the voice would say (FILE.script.txt) and the report of "
918
+ "every change (FILE.report.json, FILE.report.md). Lists the chapters to choose from.",
919
+ add_script,
920
+ run_script_command,
921
+ _categories_help,
922
+ ),
923
+ Command(
924
+ "convert",
925
+ "make the audiobook: an M4B with chapters, an MP3 per chapter and a read-along file",
926
+ "Make the audiobook of FILE. Running the same command again continues an interrupted "
927
+ "run: sentences already spoken are taken from the speech cache.",
928
+ add_convert,
929
+ run_convert_command,
930
+ _categories_help,
931
+ ),
932
+ Command(
933
+ "voices",
934
+ "list the voices; `voices download` installs their files",
935
+ "List the voices and whether their files are installed. `voices download` shows "
936
+ "what it fetches and asks before anything is downloaded.",
937
+ add_voices,
938
+ run_voices_command,
939
+ ),
940
+ Command(
941
+ "clean",
942
+ "remove the speech cache of FILE",
943
+ "Remove the speech cache of FILE (the spoken sentences kept for a quick rerun). The "
944
+ "audiobook and the records stay.",
945
+ add_clean,
946
+ run_clean_command,
947
+ ),
948
+ Command(
949
+ "ui",
950
+ "open the sci2audio interface in your browser (it stays on this computer)",
951
+ "Start the sci2audio interface on this computer (127.0.0.1 only) and open it in "
952
+ "your browser. Press Ctrl-C to stop it.",
953
+ add_ui,
954
+ run_ui_command,
955
+ ),
956
+ )
957
+
958
+
959
+ # ---- running ----------------------------------------------------------------------------
960
+
961
+ _JOB_ERRORS: tuple[tuple[type[JobError], Exit, str], ...] = (
962
+ (UnsupportedFileError, Exit.UNSUPPORTED, ""),
963
+ (UnreadableFileError, Exit.UNREADABLE, ""),
964
+ (OutputExistsError, Exit.EXISTS, " (add --overwrite)"),
965
+ (JobBusyError, Exit.BUSY, ""),
966
+ (JobCancelledError, Exit.INTERRUPTED, "; run the same command again to continue"),
967
+ )
968
+
969
+
970
+ def _job_error(error: JobError) -> tuple[str, Exit]:
971
+ for kind, code, hint in _JOB_ERRORS:
972
+ if isinstance(error, kind):
973
+ return f"{error}{hint}", code
974
+ return str(error), Exit.FAILED
975
+
976
+
977
+ def run(argv: Sequence[str] | None, env: CliEnv) -> int:
978
+ """Parse `argv`, run the command and return its exit code (argparse may exit with 0
979
+ for --help and --version, or 2 for a bad option)."""
980
+ parser = build_parser(env)
981
+ args = parser.parse_args(argv)
982
+ if args.command is None:
983
+ parser.print_help(env.err)
984
+ return int(Exit.USAGE)
985
+ try:
986
+ return int(args.run(args, env))
987
+ except CliError as error:
988
+ return _fail(env, str(error), error.code)
989
+ except JobError as error:
990
+ message, code = _job_error(error)
991
+ return _fail(env, message, code)
992
+ except KeyboardInterrupt:
993
+ return _fail(env, "stopped; run the same command again to continue", Exit.INTERRUPTED)
994
+ except Exception as error: # a bug: one line, never a traceback (D-212)
995
+ return _fail(
996
+ env,
997
+ f"unexpected error, this is a bug in sci2audio ({type(error).__name__}: {error})",
998
+ Exit.FAILED,
999
+ )
1000
+
1001
+
1002
+ def _fail(env: CliEnv, message: str, code: Exit) -> int:
1003
+ env.err.write(f"{PROG}: {message}\n")
1004
+ env.err.flush()
1005
+ return int(code)
1006
+
1007
+
1008
+ def main(argv: Sequence[str] | None = None) -> int:
1009
+ return run(argv, CliEnv.real())
1010
+
1011
+
1012
+ if __name__ == "__main__":
1013
+ raise SystemExit(main())