sci2audio 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. sci2audio/__init__.py +3 -0
  2. sci2audio/app/__init__.py +110 -0
  3. sci2audio/app/audio.py +847 -0
  4. sci2audio/app/chapters.py +109 -0
  5. sci2audio/app/chunks.py +102 -0
  6. sci2audio/app/config.py +136 -0
  7. sci2audio/app/estimate.py +192 -0
  8. sci2audio/app/files.py +98 -0
  9. sci2audio/app/job.py +285 -0
  10. sci2audio/app/lock.py +92 -0
  11. sci2audio/app/report.py +400 -0
  12. sci2audio/app/runner.py +92 -0
  13. sci2audio/app/script.py +95 -0
  14. sci2audio/app/settings.py +58 -0
  15. sci2audio/app/voice.py +70 -0
  16. sci2audio/cli.py +1013 -0
  17. sci2audio/core/__init__.py +81 -0
  18. sci2audio/core/audio.py +39 -0
  19. sci2audio/core/content.py +266 -0
  20. sci2audio/core/document.py +225 -0
  21. sci2audio/core/edits.py +121 -0
  22. sci2audio/core/export.py +76 -0
  23. sci2audio/extraction/__init__.py +28 -0
  24. sci2audio/extraction/cover.py +68 -0
  25. sci2audio/extraction/epub.py +576 -0
  26. sci2audio/extraction/errors.py +7 -0
  27. sci2audio/extraction/markdown.py +250 -0
  28. sci2audio/extraction/markup.py +356 -0
  29. sci2audio/extraction/pdf.py +623 -0
  30. sci2audio/extraction/text.py +136 -0
  31. sci2audio/layout/__init__.py +13 -0
  32. sci2audio/layout/build.py +265 -0
  33. sci2audio/layout/captions.py +77 -0
  34. sci2audio/layout/footnotes.py +180 -0
  35. sci2audio/layout/formulas.py +392 -0
  36. sci2audio/layout/furniture.py +326 -0
  37. sci2audio/layout/geometry.py +124 -0
  38. sci2audio/layout/graphics.py +569 -0
  39. sci2audio/layout/headings.py +358 -0
  40. sci2audio/layout/lists.py +167 -0
  41. sci2audio/layout/maths.py +47 -0
  42. sci2audio/layout/order.py +374 -0
  43. sci2audio/layout/paragraphs.py +518 -0
  44. sci2audio/layout/settings.py +142 -0
  45. sci2audio/layout/split.py +113 -0
  46. sci2audio/layout/tagged.py +442 -0
  47. sci2audio/output/__init__.py +44 -0
  48. sci2audio/output/ffmpeg.py +111 -0
  49. sci2audio/output/model.py +125 -0
  50. sci2audio/output/names.py +70 -0
  51. sci2audio/output/readalong.py +160 -0
  52. sci2audio/output/timeline.py +113 -0
  53. sci2audio/output/wav.py +83 -0
  54. sci2audio/output/write.py +280 -0
  55. sci2audio/testing/__init__.py +32 -0
  56. sci2audio/testing/documents.py +124 -0
  57. sci2audio/testing/regression.py +197 -0
  58. sci2audio/testing/text_steps.py +262 -0
  59. sci2audio/testing/voices.py +164 -0
  60. sci2audio/textprep/__init__.py +105 -0
  61. sci2audio/textprep/abbreviations.py +138 -0
  62. sci2audio/textprep/characters.py +267 -0
  63. sci2audio/textprep/citations.py +683 -0
  64. sci2audio/textprep/data/en/abbreviations.toml +209 -0
  65. sci2audio/textprep/data/en/units.toml +548 -0
  66. sci2audio/textprep/footnotes.py +418 -0
  67. sci2audio/textprep/furniture.py +46 -0
  68. sci2audio/textprep/hyphens.py +228 -0
  69. sci2audio/textprep/links.py +628 -0
  70. sci2audio/textprep/notation.py +170 -0
  71. sci2audio/textprep/numbers.py +529 -0
  72. sci2audio/textprep/placeholders.py +68 -0
  73. sci2audio/textprep/references.py +107 -0
  74. sci2audio/textprep/registry.py +83 -0
  75. sci2audio/textprep/runner.py +206 -0
  76. sci2audio/textprep/spoken.py +33 -0
  77. sci2audio/textprep/statistics.py +322 -0
  78. sci2audio/textprep/steps.py +153 -0
  79. sci2audio/textprep/units.py +449 -0
  80. sci2audio/textprep/utterances.py +315 -0
  81. sci2audio/textprep/wording.py +93 -0
  82. sci2audio/textprep/wordlists.py +189 -0
  83. sci2audio/voices/__init__.py +102 -0
  84. sci2audio/voices/download.py +340 -0
  85. sci2audio/voices/engine.py +136 -0
  86. sci2audio/voices/errors.py +88 -0
  87. sci2audio/voices/kokoro_engine.py +346 -0
  88. sci2audio/voices/runtime.py +37 -0
  89. sci2audio/voices/types.py +206 -0
  90. sci2audio/voices/wav.py +82 -0
  91. sci2audio/web/__init__.py +13 -0
  92. sci2audio/web/api.py +658 -0
  93. sci2audio/web/config.py +97 -0
  94. sci2audio/web/documents.py +496 -0
  95. sci2audio/web/i18n.py +188 -0
  96. sci2audio/web/jobs.py +265 -0
  97. sci2audio/web/launch.py +121 -0
  98. sci2audio/web/limits.py +64 -0
  99. sci2audio/web/models.py +178 -0
  100. sci2audio/web/pages.py +509 -0
  101. sci2audio/web/player.py +549 -0
  102. sci2audio/web/security.py +194 -0
  103. sci2audio/web/server.py +116 -0
  104. sci2audio/web/state.py +88 -0
  105. sci2audio/web/static/app.css +150 -0
  106. sci2audio/web/static/app.js +499 -0
  107. sci2audio/web/static/player.js +277 -0
  108. sci2audio/web/strings/de.toml +348 -0
  109. sci2audio/web/strings/en.toml +343 -0
  110. sci2audio/web/tasks.py +280 -0
  111. sci2audio/web/templates/_remove.html +17 -0
  112. sci2audio/web/templates/base.html +32 -0
  113. sci2audio/web/templates/document.html +153 -0
  114. sci2audio/web/templates/index.html +41 -0
  115. sci2audio/web/templates/job.html +66 -0
  116. sci2audio/web/templates/listen.html +75 -0
  117. sci2audio/web/templates/missing.html +7 -0
  118. sci2audio/web/templates/privacy.html +47 -0
  119. sci2audio/web/templates/voice.html +31 -0
  120. sci2audio-1.0.0.dist-info/METADATA +171 -0
  121. sci2audio-1.0.0.dist-info/RECORD +125 -0
  122. sci2audio-1.0.0.dist-info/WHEEL +4 -0
  123. sci2audio-1.0.0.dist-info/entry_points.txt +2 -0
  124. sci2audio-1.0.0.dist-info/licenses/LICENSE +674 -0
  125. sci2audio-1.0.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +86 -0
sci2audio/app/job.py ADDED
@@ -0,0 +1,285 @@
1
+ """The script job: a file in, the listening script and the pre-analysis report out (#70).
2
+
3
+ file → extraction (picked by file type) → layout → text steps on the whole document →
4
+ chapters → utterances of the chosen chapters → script and report. The job gets its
5
+ settings and its working folder passed in and keeps no state between jobs. It never stops
6
+ to ask (D-61 (3)): a failing text step, an `ask` edit or an unknown chapter is reported and
7
+ the run goes on. It fails, with a `JobError` a person can act on and nothing half-written,
8
+ only when it cannot go on: a file type it can't read, a file it can't open or that is
9
+ damaged, or a write that fails (a full disk).
10
+
11
+ Files: `<name>.script.txt`, `<name>.report.json` and `<name>.report.md` in the output folder
12
+ (`<name>` is the source's file name without its extension), and `job.json` in the working
13
+ folder, written last: the record the audio job (#71) resumes from.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ from collections.abc import Callable, Mapping, Sequence
20
+ from dataclasses import dataclass
21
+ from enum import StrEnum
22
+ from pathlib import Path, PurePath
23
+ from typing import Any
24
+
25
+ from sci2audio import __version__
26
+ from sci2audio.app.files import write_text_atomically, write_texts
27
+ from sci2audio.app.report import (
28
+ Report,
29
+ build_report,
30
+ report_data,
31
+ report_markdown,
32
+ script_text,
33
+ )
34
+ from sci2audio.app.script import ScriptResult, prepare_script
35
+ from sci2audio.app.settings import JobSettings
36
+ from sci2audio.core import Content, Document
37
+ from sci2audio.extraction import (
38
+ Cover,
39
+ ExtractionError,
40
+ extract_epub,
41
+ extract_epub_cover,
42
+ extract_markdown,
43
+ extract_pdf,
44
+ extract_pdf_cover,
45
+ extract_text,
46
+ )
47
+ from sci2audio.layout import LayoutSettings, build_document
48
+ from sci2audio.output.names import clean
49
+ from sci2audio.textprep import BUILT_IN_STEPS, DEFAULT_MAX_CHARS, TextStep
50
+
51
+ RECORD = "job.json"
52
+ RECORD_FORMAT = "sci2audio-job"
53
+ RECORD_VERSION = 1
54
+
55
+ # extraction by file suffix (D-350); pasted text is kept as a .txt file, so it reads as text
56
+ EXTRACTORS: Mapping[str, Callable[[Path], Content]] = {
57
+ ".epub": extract_epub,
58
+ ".markdown": extract_markdown,
59
+ ".md": extract_markdown,
60
+ ".pdf": extract_pdf,
61
+ ".txt": extract_text,
62
+ }
63
+ # the cover image a file type gives, or None (D-313); a type without one has no entry
64
+ COVERS: Mapping[str, Callable[[Path], Cover | None]] = {
65
+ ".epub": extract_epub_cover,
66
+ ".pdf": extract_pdf_cover,
67
+ }
68
+
69
+
70
+ class Phase(StrEnum):
71
+ """What the job is doing; `progress` is called as each one starts."""
72
+
73
+ READING = "reading" # extraction
74
+ LAYOUT = "layout"
75
+ TEXT_STEPS = "text-steps" # the steps, chapters and cutting into utterances
76
+ LOADING_VOICE = "loading-voice" # audio job: the voice model loads (skipped if all cached)
77
+ SPEAKING = "speaking" # audio job: the voice speaks the chunks not yet in the cache
78
+ ENCODING = "encoding" # audio job: output writes the MP3s, the M4B and the read-along
79
+ WRITING = "writing" # script job: its three files
80
+ DONE = "done"
81
+
82
+
83
+ class JobError(Exception):
84
+ """The job could not go on; the message says why in words a person can act on."""
85
+
86
+
87
+ class UnsupportedFileError(JobError):
88
+ """A file type the job can't read."""
89
+
90
+
91
+ class UnreadableFileError(JobError):
92
+ """A file of a type the job reads that can't be opened or is damaged (D-223)."""
93
+
94
+
95
+ class OutputExistsError(JobError):
96
+ """An output file exists and `overwrite` was not given."""
97
+
98
+
99
+ class JobBusyError(JobError):
100
+ """The runner, or another process on the same working folder, is already running a job
101
+ (one job at a time)."""
102
+
103
+
104
+ class JobCancelledError(JobError):
105
+ """The person cancelled the job; what was finished is kept for a rerun."""
106
+
107
+
108
+ @dataclass(frozen=True)
109
+ class ScriptFiles:
110
+ """Where the script job writes."""
111
+
112
+ script: Path
113
+ report_json: Path
114
+ report_markdown: Path
115
+ record: Path # in the working folder
116
+
117
+ @property
118
+ def outputs(self) -> tuple[Path, Path, Path]:
119
+ return (self.script, self.report_json, self.report_markdown)
120
+
121
+
122
+ @dataclass(frozen=True)
123
+ class ScriptJob:
124
+ """What a finished script job worked out and wrote."""
125
+
126
+ result: ScriptResult
127
+ report: Report
128
+ files: ScriptFiles
129
+
130
+
131
+ def script_files(source: Path, settings: JobSettings, work: Path) -> ScriptFiles:
132
+ """The files a script job of `source` writes."""
133
+ folder = settings.output_folder or source.absolute().parent
134
+ name = clean(PurePath(source.name).stem, "document")
135
+ return ScriptFiles(
136
+ script=folder / f"{name}.script.txt",
137
+ report_json=folder / f"{name}.report.json",
138
+ report_markdown=folder / f"{name}.report.md",
139
+ record=work / RECORD,
140
+ )
141
+
142
+
143
+ def settings_data(settings: JobSettings) -> dict[str, Any]:
144
+ """The settings as the report and the job record keep them (no folders: those are
145
+ machine-specific and not part of what the job decides)."""
146
+ return {
147
+ "language": settings.language,
148
+ "switched_off": sorted(str(category) for category in settings.switched_off),
149
+ "chapters": None if settings.chapters is None else sorted(settings.chapters),
150
+ "voice": settings.voice,
151
+ "speed": settings.speed,
152
+ }
153
+
154
+
155
+ def run_script_job(
156
+ source: Path,
157
+ settings: JobSettings,
158
+ work: Path,
159
+ *,
160
+ overwrite: bool = False,
161
+ progress: Callable[[Phase], None] | None = None,
162
+ steps: Sequence[TextStep] = BUILT_IN_STEPS,
163
+ max_chars: int = DEFAULT_MAX_CHARS,
164
+ ) -> ScriptJob:
165
+ """Read `source`, prepare its listening script and report, and write them.
166
+
167
+ `work` is the job's own working folder (made if missing). Existing output files are
168
+ replaced only with `overwrite`; that is checked before any work starts. Raises
169
+ JobError (or a subclass) when the job cannot go on; nothing half-written is left.
170
+ """
171
+ report: Callable[[Phase], None] = progress or (lambda phase: None)
172
+ check_readable(source)
173
+ files = script_files(source, settings, work)
174
+ check_folders((files.script.parent, files.record.parent))
175
+ if not overwrite and (existing := [path for path in files.outputs if path.exists()]):
176
+ names = ", ".join(path.name for path in existing)
177
+ raise OutputExistsError(f"these files exist already: {names}; choose to replace them")
178
+
179
+ document = read_document(source, settings, report)
180
+
181
+ report(Phase.TEXT_STEPS)
182
+ result = prepare_script(document, settings, steps, max_chars)
183
+ job_report = build_report(result, settings_data(settings))
184
+
185
+ report(Phase.WRITING)
186
+ _write(files, result, job_report, settings, overwrite)
187
+ report(Phase.DONE)
188
+ return ScriptJob(result, job_report, files)
189
+
190
+
191
+ def _write(
192
+ files: ScriptFiles,
193
+ result: ScriptResult,
194
+ job_report: Report,
195
+ settings: JobSettings,
196
+ overwrite: bool,
197
+ ) -> None:
198
+ texts = {
199
+ files.script: script_text(result),
200
+ files.report_json: json_text(report_data(job_report)),
201
+ files.report_markdown: report_markdown(job_report),
202
+ }
203
+ for folder in (files.script.parent, files.record.parent):
204
+ try:
205
+ folder.mkdir(parents=True, exist_ok=True)
206
+ except OSError as error:
207
+ raise _folder_error(folder, error) from error
208
+ try:
209
+ write_texts(texts, overwrite=overwrite)
210
+ record = _record(files, result, settings)
211
+ write_text_atomically(files.record, json_text(record), overwrite=True)
212
+ except FileExistsError as error:
213
+ target = error.filename2 or error.filename
214
+ name = PurePath(str(target)).name if target else "an output file"
215
+ raise OutputExistsError(f"{name} appeared during the run; choose to replace it") from error
216
+ except OSError as error:
217
+ where = PurePath(str(error.filename)).name if error.filename else "the output"
218
+ raise JobError(f"could not write {where}: {error.strerror or error}") from error
219
+
220
+
221
+ def check_readable(source: Path) -> None:
222
+ """Refuse before any work a file type the job can't read."""
223
+ if source.suffix.lower() not in EXTRACTORS:
224
+ known = ", ".join(sorted(EXTRACTORS))
225
+ raise UnsupportedFileError(f"{source.name}: sci2audio can read {known} files only")
226
+
227
+
228
+ def read_document(source: Path, settings: JobSettings, report: Callable[[Phase], None]) -> Document:
229
+ """Extraction (picked by file type) and layout; JobError if the file can't be read."""
230
+ check_readable(source)
231
+ extract = EXTRACTORS[source.suffix.lower()]
232
+ report(Phase.READING)
233
+ try:
234
+ content = extract(source)
235
+ except ExtractionError as error:
236
+ raise UnreadableFileError(str(error)) from error
237
+ except OSError as error:
238
+ raise UnreadableFileError(f"{source.name}: cannot be opened ({error.strerror})") from error
239
+ report(Phase.LAYOUT)
240
+ return build_document(content, LayoutSettings(language=settings.language))
241
+
242
+
243
+ def check_folders(folders: Sequence[Path]) -> None:
244
+ """Refuse before any work an output or working folder that is a file."""
245
+ for folder in folders:
246
+ for path in (folder, *folder.parents):
247
+ if path.exists():
248
+ if not path.is_dir():
249
+ raise JobError(f"{path.name} is not a folder; choose another output folder")
250
+ break
251
+
252
+
253
+ def _folder_error(folder: Path, error: OSError) -> JobError:
254
+ if isinstance(error, FileExistsError | NotADirectoryError):
255
+ return JobError(f"{folder.name} is not a folder; choose another output folder")
256
+ return JobError(f"could not make the folder {folder.name}: {error.strerror or error}")
257
+
258
+
259
+ def _record(files: ScriptFiles, result: ScriptResult, settings: JobSettings) -> dict[str, Any]:
260
+ source = result.document.content.source
261
+ return {
262
+ "format": RECORD_FORMAT,
263
+ "version": RECORD_VERSION,
264
+ "sci2audio": __version__,
265
+ "source": {"name": source.name, "format": str(source.format), "sha256": source.sha256},
266
+ "settings": settings_data(settings),
267
+ "chapters": [
268
+ {
269
+ "index": chapter.chapter.index,
270
+ "title": chapter.chapter.title,
271
+ "selected": chapter.selected,
272
+ "utterances": chapter.spoken,
273
+ }
274
+ for chapter in result.chapters
275
+ ],
276
+ "files": {
277
+ "script": files.script.name,
278
+ "report_json": files.report_json.name,
279
+ "report_markdown": files.report_markdown.name,
280
+ },
281
+ }
282
+
283
+
284
+ def json_text(data: Any) -> str:
285
+ return json.dumps(data, ensure_ascii=False, indent=2) + "\n"
sci2audio/app/lock.py ADDED
@@ -0,0 +1,92 @@
1
+ """One job per working folder, also across processes (D-161).
2
+
3
+ `work_folder_lock` holds an operating-system lock on `<work>/job.lock` while a job runs:
4
+ `fcntl.flock` on Linux and macOS, `msvcrt.locking` on Windows. The system releases it when
5
+ the process ends, even when it is killed, so a crashed job never leaves a stale lock. The
6
+ lock file itself stays; it holds nothing. Windows may take a moment to free the lock of a
7
+ killed process ("depends upon available system resources", msvcrt docs), so there a busy
8
+ lock is tried again for up to `WINDOWS_GRACE` seconds before the job is refused.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import contextlib
14
+ import sys
15
+ import time
16
+ from collections.abc import Iterator
17
+ from pathlib import Path
18
+ from typing import IO
19
+
20
+ from sci2audio.app.job import JobBusyError, JobError
21
+
22
+ LOCK_NAME = "job.lock"
23
+ WINDOWS_GRACE = 3.0 # seconds
24
+
25
+
26
+ def _try_lock(file: IO[bytes]) -> bool:
27
+ if sys.platform == "win32":
28
+ import msvcrt
29
+
30
+ file.seek(0)
31
+ try:
32
+ msvcrt.locking(file.fileno(), msvcrt.LK_NBLCK, 1)
33
+ except OSError:
34
+ return False
35
+ return True
36
+ import fcntl
37
+
38
+ try:
39
+ fcntl.flock(file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
40
+ except OSError: # BlockingIOError: someone else holds it
41
+ return False
42
+ return True
43
+
44
+
45
+ def _lock_soon(file: IO[bytes]) -> bool:
46
+ if _try_lock(file):
47
+ return True
48
+ if sys.platform != "win32":
49
+ return False
50
+ deadline = time.monotonic() + WINDOWS_GRACE
51
+ while time.monotonic() < deadline:
52
+ time.sleep(0.1)
53
+ if _try_lock(file):
54
+ return True
55
+ return False
56
+
57
+
58
+ def _unlock(file: IO[bytes]) -> None:
59
+ if sys.platform == "win32":
60
+ import msvcrt
61
+
62
+ file.seek(0)
63
+ msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 1)
64
+ else:
65
+ import fcntl
66
+
67
+ fcntl.flock(file.fileno(), fcntl.LOCK_UN)
68
+
69
+
70
+ @contextlib.contextmanager
71
+ def work_folder_lock(work: Path) -> Iterator[None]:
72
+ """Hold the working folder's lock; JobBusyError if another job holds it."""
73
+ try:
74
+ work.mkdir(parents=True, exist_ok=True)
75
+ file = (work / LOCK_NAME).open("a+b")
76
+ except OSError as error:
77
+ raise JobError(
78
+ f"could not use the working folder {work.name}: {error.strerror or error}"
79
+ ) from error
80
+ try:
81
+ if not _lock_soon(file):
82
+ raise JobBusyError(
83
+ "another sci2audio job is working on this file's working folder; "
84
+ "wait until it has finished"
85
+ )
86
+ try:
87
+ yield
88
+ finally:
89
+ with contextlib.suppress(OSError):
90
+ _unlock(file)
91
+ finally:
92
+ file.close()