sci2audio 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sci2audio/__init__.py +3 -0
- sci2audio/app/__init__.py +110 -0
- sci2audio/app/audio.py +847 -0
- sci2audio/app/chapters.py +109 -0
- sci2audio/app/chunks.py +102 -0
- sci2audio/app/config.py +136 -0
- sci2audio/app/estimate.py +192 -0
- sci2audio/app/files.py +98 -0
- sci2audio/app/job.py +285 -0
- sci2audio/app/lock.py +92 -0
- sci2audio/app/report.py +400 -0
- sci2audio/app/runner.py +92 -0
- sci2audio/app/script.py +95 -0
- sci2audio/app/settings.py +58 -0
- sci2audio/app/voice.py +70 -0
- sci2audio/cli.py +1013 -0
- sci2audio/core/__init__.py +81 -0
- sci2audio/core/audio.py +39 -0
- sci2audio/core/content.py +266 -0
- sci2audio/core/document.py +225 -0
- sci2audio/core/edits.py +121 -0
- sci2audio/core/export.py +76 -0
- sci2audio/extraction/__init__.py +28 -0
- sci2audio/extraction/cover.py +68 -0
- sci2audio/extraction/epub.py +576 -0
- sci2audio/extraction/errors.py +7 -0
- sci2audio/extraction/markdown.py +250 -0
- sci2audio/extraction/markup.py +356 -0
- sci2audio/extraction/pdf.py +623 -0
- sci2audio/extraction/text.py +136 -0
- sci2audio/layout/__init__.py +13 -0
- sci2audio/layout/build.py +265 -0
- sci2audio/layout/captions.py +77 -0
- sci2audio/layout/footnotes.py +180 -0
- sci2audio/layout/formulas.py +392 -0
- sci2audio/layout/furniture.py +326 -0
- sci2audio/layout/geometry.py +124 -0
- sci2audio/layout/graphics.py +569 -0
- sci2audio/layout/headings.py +358 -0
- sci2audio/layout/lists.py +167 -0
- sci2audio/layout/maths.py +47 -0
- sci2audio/layout/order.py +374 -0
- sci2audio/layout/paragraphs.py +518 -0
- sci2audio/layout/settings.py +142 -0
- sci2audio/layout/split.py +113 -0
- sci2audio/layout/tagged.py +442 -0
- sci2audio/output/__init__.py +44 -0
- sci2audio/output/ffmpeg.py +111 -0
- sci2audio/output/model.py +125 -0
- sci2audio/output/names.py +70 -0
- sci2audio/output/readalong.py +160 -0
- sci2audio/output/timeline.py +113 -0
- sci2audio/output/wav.py +83 -0
- sci2audio/output/write.py +280 -0
- sci2audio/testing/__init__.py +32 -0
- sci2audio/testing/documents.py +124 -0
- sci2audio/testing/regression.py +197 -0
- sci2audio/testing/text_steps.py +262 -0
- sci2audio/testing/voices.py +164 -0
- sci2audio/textprep/__init__.py +105 -0
- sci2audio/textprep/abbreviations.py +138 -0
- sci2audio/textprep/characters.py +267 -0
- sci2audio/textprep/citations.py +683 -0
- sci2audio/textprep/data/en/abbreviations.toml +209 -0
- sci2audio/textprep/data/en/units.toml +548 -0
- sci2audio/textprep/footnotes.py +418 -0
- sci2audio/textprep/furniture.py +46 -0
- sci2audio/textprep/hyphens.py +228 -0
- sci2audio/textprep/links.py +628 -0
- sci2audio/textprep/notation.py +170 -0
- sci2audio/textprep/numbers.py +529 -0
- sci2audio/textprep/placeholders.py +68 -0
- sci2audio/textprep/references.py +107 -0
- sci2audio/textprep/registry.py +83 -0
- sci2audio/textprep/runner.py +206 -0
- sci2audio/textprep/spoken.py +33 -0
- sci2audio/textprep/statistics.py +322 -0
- sci2audio/textprep/steps.py +153 -0
- sci2audio/textprep/units.py +449 -0
- sci2audio/textprep/utterances.py +315 -0
- sci2audio/textprep/wording.py +93 -0
- sci2audio/textprep/wordlists.py +189 -0
- sci2audio/voices/__init__.py +102 -0
- sci2audio/voices/download.py +340 -0
- sci2audio/voices/engine.py +136 -0
- sci2audio/voices/errors.py +88 -0
- sci2audio/voices/kokoro_engine.py +346 -0
- sci2audio/voices/runtime.py +37 -0
- sci2audio/voices/types.py +206 -0
- sci2audio/voices/wav.py +82 -0
- sci2audio/web/__init__.py +13 -0
- sci2audio/web/api.py +658 -0
- sci2audio/web/config.py +97 -0
- sci2audio/web/documents.py +496 -0
- sci2audio/web/i18n.py +188 -0
- sci2audio/web/jobs.py +265 -0
- sci2audio/web/launch.py +121 -0
- sci2audio/web/limits.py +64 -0
- sci2audio/web/models.py +178 -0
- sci2audio/web/pages.py +509 -0
- sci2audio/web/player.py +549 -0
- sci2audio/web/security.py +194 -0
- sci2audio/web/server.py +116 -0
- sci2audio/web/state.py +88 -0
- sci2audio/web/static/app.css +150 -0
- sci2audio/web/static/app.js +499 -0
- sci2audio/web/static/player.js +277 -0
- sci2audio/web/strings/de.toml +348 -0
- sci2audio/web/strings/en.toml +343 -0
- sci2audio/web/tasks.py +280 -0
- sci2audio/web/templates/_remove.html +17 -0
- sci2audio/web/templates/base.html +32 -0
- sci2audio/web/templates/document.html +153 -0
- sci2audio/web/templates/index.html +41 -0
- sci2audio/web/templates/job.html +66 -0
- sci2audio/web/templates/listen.html +75 -0
- sci2audio/web/templates/missing.html +7 -0
- sci2audio/web/templates/privacy.html +47 -0
- sci2audio/web/templates/voice.html +31 -0
- sci2audio-1.0.0.dist-info/METADATA +171 -0
- sci2audio-1.0.0.dist-info/RECORD +125 -0
- sci2audio-1.0.0.dist-info/WHEEL +4 -0
- sci2audio-1.0.0.dist-info/entry_points.txt +2 -0
- sci2audio-1.0.0.dist-info/licenses/LICENSE +674 -0
- sci2audio-1.0.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +86 -0
sci2audio/web/api.py
ADDED
|
@@ -0,0 +1,658 @@
|
|
|
1
|
+
"""The JSON API under `/api/v1` (D-197): documents, jobs, settings and voices.
|
|
2
|
+
|
|
3
|
+
Every error answers `{"error": <kind>, "message": <words>}` (`models.ErrorOut`).
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
import json
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import AsyncIterator
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Annotated, Any
|
|
14
|
+
from urllib.parse import urlsplit
|
|
15
|
+
|
|
16
|
+
from fastapi import APIRouter, File, Query, UploadFile
|
|
17
|
+
from fastapi.responses import FileResponse, StreamingResponse
|
|
18
|
+
|
|
19
|
+
from sci2audio.app import JobBusyError, JobSettings, clean_work_folder, load_settings, save_settings
|
|
20
|
+
from sci2audio.app.settings import MAX_SPEED, MIN_SPEED
|
|
21
|
+
from sci2audio.textprep import Category
|
|
22
|
+
from sci2audio.voices import DEFAULT_VOICE, ENGLISH_VOICES, download_plan, missing_files
|
|
23
|
+
from sci2audio.web import models
|
|
24
|
+
from sci2audio.web.documents import DocumentBusyError, RemoveError, StoredDocument, StoreError
|
|
25
|
+
from sci2audio.web.jobs import BoardBusyError, JobKind, JobRecord, JobState
|
|
26
|
+
from sci2audio.web.player import audiobook_file, recorded_folder
|
|
27
|
+
from sci2audio.web.state import ApiError, ServerState, state_of
|
|
28
|
+
from sci2audio.web.tasks import plan_digest, script_work, voices_work
|
|
29
|
+
|
|
30
|
+
router = APIRouter(prefix="/api/v1")
|
|
31
|
+
|
|
32
|
+
KEEP_ALIVE = 15.0 # seconds between comment lines on an idle event stream (D-201)
|
|
33
|
+
POLL = 0.1 # seconds between looks at a job's progress
|
|
34
|
+
# hosts the voice download talks to, with where they redirect (D-141)
|
|
35
|
+
REDIRECTS = {
|
|
36
|
+
"huggingface.co": "Hugging Face's download servers (its CDN and LFS/Xet storage)",
|
|
37
|
+
"github.com": "objects.githubusercontent.com (GitHub's download server)",
|
|
38
|
+
}
|
|
39
|
+
MEDIA_TYPES = {
|
|
40
|
+
".wav": "audio/wav",
|
|
41
|
+
".m4b": "audio/mp4",
|
|
42
|
+
".mp3": "audio/mpeg",
|
|
43
|
+
".json": "application/json",
|
|
44
|
+
".txt": "text/plain; charset=utf-8",
|
|
45
|
+
".md": "text/markdown; charset=utf-8",
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# Our own words for answers whose standard phrase reads like a leaked server error: the
|
|
50
|
+
# OpenAPI document must never say "Internal Server Error" (the ZAP baseline scan flags it
|
|
51
|
+
# as error disclosure, alerts 90022 and 10023; #120 review round 3)
|
|
52
|
+
DESCRIPTIONS = {500: "sci2audio could not finish; the message says what is left to do"}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _errors(*codes: int) -> dict[int | str, dict[str, Any]]:
|
|
56
|
+
"""The error answers of a route, for the OpenAPI document; every route can answer 400
|
|
57
|
+
(a body or path that can't be read) and 422 (one that doesn't fit)."""
|
|
58
|
+
answers: dict[int | str, dict[str, Any]] = {}
|
|
59
|
+
for code in sorted({*codes, 400, 422}):
|
|
60
|
+
answers[code] = {"model": models.ErrorOut}
|
|
61
|
+
if code in DESCRIPTIONS:
|
|
62
|
+
answers[code]["description"] = DESCRIPTIONS[code]
|
|
63
|
+
return answers
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
BODY = (411, 413) # from the size limit (limits.py)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
State = Annotated[ServerState, state_of]
|
|
70
|
+
Download = Annotated[
|
|
71
|
+
bool, Query(description="true: sent as an attachment to save, not shown (D-255)")
|
|
72
|
+
]
|
|
73
|
+
_FILE_ANSWERS: dict[int | str, dict[str, Any]] = {
|
|
74
|
+
200: {"content": {"application/octet-stream": {}}, "description": "the file"},
|
|
75
|
+
206: {"content": {"application/octet-stream": {}}, "description": "part of the file"},
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# --- documents -------------------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _document_out(document: StoredDocument) -> models.DocumentOut:
|
|
83
|
+
return models.DocumentOut(
|
|
84
|
+
id=document.id,
|
|
85
|
+
name=document.name,
|
|
86
|
+
size=document.size,
|
|
87
|
+
has_report=document.report_file.is_file(),
|
|
88
|
+
has_audio=document.has_audio,
|
|
89
|
+
speech_cache_bytes=document.speech_cache_bytes,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _document(state: ServerState, document_id: str) -> StoredDocument:
|
|
94
|
+
document = state.store.get(document_id)
|
|
95
|
+
if document is None:
|
|
96
|
+
raise ApiError(404, "not-found", f"no document {document_id}; open the file again")
|
|
97
|
+
return document
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _job_on(state: ServerState, document_id: str) -> str | None:
|
|
101
|
+
"""Why the document can't be changed now: this server's running job works on it."""
|
|
102
|
+
job = state.board.current
|
|
103
|
+
if job is not None and job.document == document_id:
|
|
104
|
+
return f"job {job.id} is working on this document; wait for it"
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _not_in_use(state: ServerState, document_id: str) -> None:
|
|
109
|
+
reason = _job_on(state, document_id)
|
|
110
|
+
if reason is not None:
|
|
111
|
+
raise ApiError(409, "busy", reason)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _stored(state: ServerState, store: Any) -> models.DocumentOut:
|
|
115
|
+
try:
|
|
116
|
+
document = store()
|
|
117
|
+
except StoreError as error:
|
|
118
|
+
status = {"unsupported-file": 415, "too-large": 413, "disk-full": 507}.get(error.kind, 500)
|
|
119
|
+
raise ApiError(status, error.kind, str(error)) from error
|
|
120
|
+
return _document_out(document)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@router.post(
|
|
124
|
+
"/documents",
|
|
125
|
+
status_code=201,
|
|
126
|
+
response_model=models.DocumentOut,
|
|
127
|
+
responses=_errors(409, 415, 507, *BODY),
|
|
128
|
+
summary="Open a file",
|
|
129
|
+
)
|
|
130
|
+
def open_file(state: State, file: Annotated[UploadFile, File()]) -> models.DocumentOut:
|
|
131
|
+
name = file.filename or ""
|
|
132
|
+
current = state.board.current
|
|
133
|
+
in_use = None if current is None else current.document
|
|
134
|
+
return _stored(state, lambda: state.store.store(name, file.file, in_use=in_use))
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@router.post(
|
|
138
|
+
"/documents/text",
|
|
139
|
+
status_code=201,
|
|
140
|
+
response_model=models.DocumentOut,
|
|
141
|
+
responses=_errors(409, 415, 507, *BODY),
|
|
142
|
+
summary="Open pasted text",
|
|
143
|
+
)
|
|
144
|
+
def open_text(state: State, body: models.TextIn) -> models.DocumentOut:
|
|
145
|
+
current = state.board.current
|
|
146
|
+
in_use = None if current is None else current.document
|
|
147
|
+
return _stored(state, lambda: state.store.store_text(body.text, body.name, in_use=in_use))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@router.get(
|
|
151
|
+
"/documents",
|
|
152
|
+
response_model=list[models.DocumentOut],
|
|
153
|
+
responses=_errors(),
|
|
154
|
+
summary="Opened documents",
|
|
155
|
+
)
|
|
156
|
+
def list_documents(state: State) -> list[models.DocumentOut]:
|
|
157
|
+
return [_document_out(document) for document in state.store.all()]
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@router.delete(
|
|
161
|
+
"/documents/{document_id}",
|
|
162
|
+
response_model=models.RemovedOut,
|
|
163
|
+
responses=_errors(404, 409, 500),
|
|
164
|
+
summary="Remove a document and its whole working folder",
|
|
165
|
+
)
|
|
166
|
+
def remove_document(state: State, document_id: str) -> models.RemovedOut:
|
|
167
|
+
"""Removes the copy of the file, its analysis, the speech cache and the records; never
|
|
168
|
+
anything in an output folder, so the audiobook stays (#120, D-353 to D-358). Refused
|
|
169
|
+
with 409 while a job works on the document; 500 names what could not be removed, and
|
|
170
|
+
the document then stays listed for another try."""
|
|
171
|
+
document = _document(state, document_id)
|
|
172
|
+
folder = recorded_folder(document.folder, document.id)
|
|
173
|
+
try:
|
|
174
|
+
removed = state.store.remove(document_id, busy=lambda: _job_on(state, document_id))
|
|
175
|
+
except DocumentBusyError as error:
|
|
176
|
+
raise ApiError(409, "busy", str(error)) from error
|
|
177
|
+
except RemoveError as error:
|
|
178
|
+
raise ApiError(500, "remove-failed", str(error)) from error
|
|
179
|
+
if removed is None: # removed meanwhile by another request
|
|
180
|
+
raise ApiError(404, "not-found", f"no document {document_id}; open the file again")
|
|
181
|
+
state.board.forget(document_id)
|
|
182
|
+
return models.RemovedOut(
|
|
183
|
+
id=removed.id,
|
|
184
|
+
name=removed.name,
|
|
185
|
+
freed_bytes=removed.freed_bytes,
|
|
186
|
+
audiobook_folder=None if folder is None else str(folder),
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@router.get(
|
|
191
|
+
"/documents/{document_id}",
|
|
192
|
+
response_model=models.DocumentOut,
|
|
193
|
+
responses=_errors(404),
|
|
194
|
+
summary="One document",
|
|
195
|
+
)
|
|
196
|
+
def get_document(state: State, document_id: str) -> models.DocumentOut:
|
|
197
|
+
return _document_out(_document(state, document_id))
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _report(state: ServerState, document_id: str) -> dict[str, Any]:
|
|
201
|
+
document = _document(state, document_id)
|
|
202
|
+
try:
|
|
203
|
+
data = json.loads(document.report_file.read_text(encoding="utf-8"))
|
|
204
|
+
except (OSError, ValueError) as error:
|
|
205
|
+
raise ApiError(
|
|
206
|
+
404, "no-report", f"{document.name} has no pre-analysis yet; start a script job"
|
|
207
|
+
) from error
|
|
208
|
+
if not isinstance(data, dict):
|
|
209
|
+
raise ApiError(404, "no-report", f"{document.name} has no readable pre-analysis")
|
|
210
|
+
return data
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
@router.get(
|
|
214
|
+
"/documents/{document_id}/report",
|
|
215
|
+
response_model=dict[str, Any],
|
|
216
|
+
responses=_errors(404),
|
|
217
|
+
summary="The latest pre-analysis report (sci2audio-report v1)",
|
|
218
|
+
)
|
|
219
|
+
def get_report(state: State, document_id: str) -> dict[str, Any]:
|
|
220
|
+
return _report(state, document_id)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@router.get(
|
|
224
|
+
"/documents/{document_id}/chapters",
|
|
225
|
+
response_model=list[models.ChapterOut],
|
|
226
|
+
responses=_errors(404),
|
|
227
|
+
summary="The chapters of the latest pre-analysis",
|
|
228
|
+
)
|
|
229
|
+
def get_chapters(state: State, document_id: str) -> list[models.ChapterOut]:
|
|
230
|
+
chapters = _report(state, document_id).get("chapters", [])
|
|
231
|
+
return [models.ChapterOut.model_validate(chapter) for chapter in chapters]
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@router.get(
|
|
235
|
+
"/documents/{document_id}/files/{name}",
|
|
236
|
+
response_class=FileResponse,
|
|
237
|
+
responses={**_FILE_ANSWERS, **_errors(404)},
|
|
238
|
+
summary="A file of the latest pre-analysis: the script or the report",
|
|
239
|
+
)
|
|
240
|
+
def document_file(
|
|
241
|
+
state: State, document_id: str, name: str, download: Download = False
|
|
242
|
+
) -> FileResponse:
|
|
243
|
+
"""The listening script, `report.json` or `report.md` of the latest pre-analysis, by
|
|
244
|
+
name as written (D-255); they outlast the job that wrote them."""
|
|
245
|
+
document = _document(state, document_id)
|
|
246
|
+
found = next((path for path in document.analysis_files() if path.name == name), None)
|
|
247
|
+
if found is None:
|
|
248
|
+
raise ApiError(404, "not-found", f"{document.name} has no analysis file {name!r}")
|
|
249
|
+
return _file(found, download)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
@router.get(
|
|
253
|
+
"/documents/{document_id}/audiobook/{name}",
|
|
254
|
+
response_class=FileResponse,
|
|
255
|
+
responses={**_FILE_ANSWERS, **_errors(404)},
|
|
256
|
+
summary="A file of the document's latest audiobook: an MP3, the M4B or the read-along",
|
|
257
|
+
)
|
|
258
|
+
def audiobook(
|
|
259
|
+
state: State, document_id: str, name: str, download: Download = False
|
|
260
|
+
) -> FileResponse:
|
|
261
|
+
"""A file the document's latest audio job wrote, by name as written, for the player
|
|
262
|
+
(#78, D-295): only names its record lists, only from the recorded, the saved or the
|
|
263
|
+
default output folder, and only once the read-along file there agrees with the
|
|
264
|
+
record. Range requests answer 206, so the player can seek."""
|
|
265
|
+
document = _document(state, document_id)
|
|
266
|
+
saved, _ = load_settings(state.config.config_file)
|
|
267
|
+
found = audiobook_file(
|
|
268
|
+
document.folder, document.id, name, saved.output_folder, state.config.output_folder
|
|
269
|
+
)
|
|
270
|
+
if found is None:
|
|
271
|
+
raise ApiError(404, "not-found", f"{document.name} has no audiobook file {name!r}")
|
|
272
|
+
return _file(found, download)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
@router.delete(
|
|
276
|
+
"/documents/{document_id}/chunks",
|
|
277
|
+
status_code=204,
|
|
278
|
+
responses=_errors(404, 409),
|
|
279
|
+
summary="Remove the kept speech chunks of a document",
|
|
280
|
+
)
|
|
281
|
+
def clean_chunks(state: State, document_id: str) -> None:
|
|
282
|
+
"""The next audio job of this document speaks everything again (D-158, D-209)."""
|
|
283
|
+
document = _document(state, document_id)
|
|
284
|
+
_not_in_use(state, document_id)
|
|
285
|
+
try:
|
|
286
|
+
clean_work_folder(document.folder)
|
|
287
|
+
except JobBusyError as error: # another process holds the working folder
|
|
288
|
+
raise ApiError(409, "busy", str(error)) from error
|
|
289
|
+
except OSError as error:
|
|
290
|
+
raise ApiError(500, "clean-failed", f"could not remove the chunks: {error}") from error
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
# --- settings --------------------------------------------------------------------------
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _settings_model(settings: JobSettings) -> models.SettingsModel:
|
|
297
|
+
return models.SettingsModel(
|
|
298
|
+
language=settings.language,
|
|
299
|
+
switched_off=sorted(settings.switched_off),
|
|
300
|
+
voice=settings.voice,
|
|
301
|
+
speed=settings.speed,
|
|
302
|
+
output_folder=None if settings.output_folder is None else str(settings.output_folder),
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _folder(text: str | None) -> Path | None:
|
|
307
|
+
if text is None or text == "":
|
|
308
|
+
return None
|
|
309
|
+
if "\0" in text:
|
|
310
|
+
raise ApiError(422, "invalid", "output_folder must not contain a NUL character")
|
|
311
|
+
folder = Path(text)
|
|
312
|
+
if not folder.is_absolute():
|
|
313
|
+
raise ApiError(422, "invalid", f"output_folder must be a full path, got {text!r}")
|
|
314
|
+
return folder
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _voice(voice: str) -> str:
|
|
318
|
+
if voice and voice not in {info.id for info in ENGLISH_VOICES}:
|
|
319
|
+
raise ApiError(422, "invalid", f"unknown voice {voice!r}")
|
|
320
|
+
return voice
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _job_settings(**fields: Any) -> JobSettings:
|
|
324
|
+
try:
|
|
325
|
+
return JobSettings(**fields)
|
|
326
|
+
except (TypeError, ValueError) as error:
|
|
327
|
+
raise ApiError(422, "invalid", str(error)) from error
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _voice_list() -> list[models.VoiceOut]:
|
|
331
|
+
return [
|
|
332
|
+
models.VoiceOut(id=v.id, name=v.name, language=v.language, gender=v.gender)
|
|
333
|
+
for v in ENGLISH_VOICES
|
|
334
|
+
]
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
@router.get(
|
|
338
|
+
"/settings",
|
|
339
|
+
response_model=models.SettingsOut,
|
|
340
|
+
responses=_errors(),
|
|
341
|
+
summary="The saved settings",
|
|
342
|
+
)
|
|
343
|
+
def get_settings(state: State) -> models.SettingsOut:
|
|
344
|
+
settings, warnings = load_settings(state.config.config_file)
|
|
345
|
+
return models.SettingsOut(
|
|
346
|
+
settings=_settings_model(settings),
|
|
347
|
+
warnings=warnings,
|
|
348
|
+
categories=[
|
|
349
|
+
models.CategoryOut(id=str(c), label=c.label, on_by_default=c.on_by_default)
|
|
350
|
+
for c in Category
|
|
351
|
+
],
|
|
352
|
+
speed=models.SpeedRange(min=MIN_SPEED, max=MAX_SPEED),
|
|
353
|
+
voices=_voice_list(),
|
|
354
|
+
default_voice=DEFAULT_VOICE,
|
|
355
|
+
default_output_folder=str(state.config.output_folder),
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
@router.put(
|
|
360
|
+
"/settings",
|
|
361
|
+
response_model=models.SettingsModel,
|
|
362
|
+
responses=_errors(507, *BODY),
|
|
363
|
+
summary="Save the settings",
|
|
364
|
+
)
|
|
365
|
+
def put_settings(state: State, body: models.SettingsModel) -> models.SettingsModel:
|
|
366
|
+
settings = _job_settings(
|
|
367
|
+
language=body.language,
|
|
368
|
+
switched_off=frozenset(body.switched_off),
|
|
369
|
+
voice=_voice(body.voice),
|
|
370
|
+
speed=body.speed,
|
|
371
|
+
output_folder=_folder(body.output_folder),
|
|
372
|
+
)
|
|
373
|
+
try:
|
|
374
|
+
save_settings(settings, state.config.config_file)
|
|
375
|
+
except OSError as error:
|
|
376
|
+
raise ApiError(
|
|
377
|
+
507, "disk-full", f"could not save the settings: {error.strerror}"
|
|
378
|
+
) from error
|
|
379
|
+
return _settings_model(settings)
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
# --- jobs ------------------------------------------------------------------------------
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _merged(saved: JobSettings, given: models.JobSettingsIn) -> JobSettings:
|
|
386
|
+
"""The saved settings with the job's own choices on top (D-200)."""
|
|
387
|
+
return _job_settings(
|
|
388
|
+
language=saved.language if given.language is None else given.language,
|
|
389
|
+
switched_off=saved.switched_off
|
|
390
|
+
if given.switched_off is None
|
|
391
|
+
else frozenset(given.switched_off),
|
|
392
|
+
chapters=None if given.chapters is None else frozenset(given.chapters),
|
|
393
|
+
voice=saved.voice if given.voice is None else _voice(given.voice),
|
|
394
|
+
speed=saved.speed if given.speed is None else given.speed,
|
|
395
|
+
output_folder=saved.output_folder
|
|
396
|
+
if given.output_folder is None
|
|
397
|
+
else _folder(given.output_folder),
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _settings_data(settings: JobSettings) -> dict[str, Any]:
|
|
402
|
+
return {
|
|
403
|
+
"language": settings.language,
|
|
404
|
+
"switched_off": sorted(str(c) for c in settings.switched_off),
|
|
405
|
+
"chapters": None if settings.chapters is None else sorted(settings.chapters),
|
|
406
|
+
"voice": settings.voice,
|
|
407
|
+
"speed": settings.speed,
|
|
408
|
+
"output_folder": None if settings.output_folder is None else str(settings.output_folder),
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _job(state: ServerState, job_id: str) -> JobRecord:
|
|
413
|
+
job = state.board.get(job_id)
|
|
414
|
+
if job is None:
|
|
415
|
+
raise ApiError(404, "not-found", f"no job {job_id} in this run of sci2audio")
|
|
416
|
+
return job
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _job_out(job: JobRecord) -> models.JobOut:
|
|
420
|
+
return models.JobOut.model_validate(job.snapshot())
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
@router.post(
|
|
424
|
+
"/jobs",
|
|
425
|
+
status_code=202,
|
|
426
|
+
response_model=models.JobOut,
|
|
427
|
+
responses=_errors(404, 409, *BODY),
|
|
428
|
+
summary="Start a job",
|
|
429
|
+
)
|
|
430
|
+
def start_job(state: State, body: models.JobIn) -> models.JobOut:
|
|
431
|
+
document = _document(state, body.document)
|
|
432
|
+
saved, _ = load_settings(state.config.config_file)
|
|
433
|
+
settings = _merged(saved, body.settings)
|
|
434
|
+
if body.kind == "audio":
|
|
435
|
+
work = state.audio_work(document, settings, body.overwrite)
|
|
436
|
+
kind = JobKind.AUDIO
|
|
437
|
+
else:
|
|
438
|
+
work = script_work(state.runner, document, settings)
|
|
439
|
+
kind = JobKind.SCRIPT
|
|
440
|
+
# the check and the start are one step against a removal (D-356): either the job runs
|
|
441
|
+
# before the removal looks for one (409 there), or the removal began and this is 404
|
|
442
|
+
with state.store.unchanged(document.id) as still:
|
|
443
|
+
if still is None:
|
|
444
|
+
raise ApiError(404, "not-found", f"no document {document.id}; open the file again")
|
|
445
|
+
try:
|
|
446
|
+
job = state.board.start(
|
|
447
|
+
kind, document.id, _settings_data(settings), work, cancellable=kind is JobKind.AUDIO
|
|
448
|
+
)
|
|
449
|
+
except BoardBusyError as error:
|
|
450
|
+
raise ApiError(409, "busy", str(error)) from error
|
|
451
|
+
return _job_out(job)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
@router.get(
|
|
455
|
+
"/jobs", response_model=list[models.JobOut], responses=_errors(), summary="Jobs of this run"
|
|
456
|
+
)
|
|
457
|
+
def list_jobs(state: State) -> list[models.JobOut]:
|
|
458
|
+
return [_job_out(job) for job in state.board.all()]
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
@router.get(
|
|
462
|
+
"/jobs/{job_id}", response_model=models.JobOut, responses=_errors(404), summary="One job"
|
|
463
|
+
)
|
|
464
|
+
def get_job(state: State, job_id: str) -> models.JobOut:
|
|
465
|
+
return _job_out(_job(state, job_id))
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
@router.post(
|
|
469
|
+
"/jobs/{job_id}/cancel",
|
|
470
|
+
status_code=202,
|
|
471
|
+
response_model=models.JobOut,
|
|
472
|
+
responses=_errors(404, 409),
|
|
473
|
+
summary="Ask a job to stop",
|
|
474
|
+
)
|
|
475
|
+
def cancel_job(state: State, job_id: str) -> models.JobOut:
|
|
476
|
+
job = _job(state, job_id)
|
|
477
|
+
if job.state is not JobState.RUNNING:
|
|
478
|
+
raise ApiError(409, "finished", f"job {job_id} has already ended")
|
|
479
|
+
if not job.cancellable:
|
|
480
|
+
raise ApiError(
|
|
481
|
+
409, "not-cancellable", "a pre-analysis takes seconds and can't be stopped; wait"
|
|
482
|
+
)
|
|
483
|
+
state.board.cancel(job_id)
|
|
484
|
+
return _job_out(job)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
_JOB = {"$ref": "#/components/schemas/JobOut"}
|
|
488
|
+
# one event of the stream (Open API 3.1: the schema of an event stream describes one event)
|
|
489
|
+
EVENT_SCHEMA: dict[str, Any] = {
|
|
490
|
+
"type": "object",
|
|
491
|
+
"required": ["event", "data"],
|
|
492
|
+
"properties": {
|
|
493
|
+
"event": {"enum": ["snapshot", "progress", "end"]},
|
|
494
|
+
"data": {"type": "string", "contentMediaType": "application/json"},
|
|
495
|
+
},
|
|
496
|
+
"if": {"properties": {"event": {"const": "progress"}}},
|
|
497
|
+
"then": {
|
|
498
|
+
"properties": {"data": {"contentSchema": {"type": ["object", "null"]}}},
|
|
499
|
+
},
|
|
500
|
+
"else": {"properties": {"data": {"contentSchema": _JOB}}},
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _event(name: str, data: Any) -> bytes:
|
|
505
|
+
return f"event: {name}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n".encode()
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
@router.get(
|
|
509
|
+
"/jobs/{job_id}/events",
|
|
510
|
+
response_class=StreamingResponse,
|
|
511
|
+
responses={
|
|
512
|
+
200: {
|
|
513
|
+
"content": {"text/event-stream": {"schema": EVENT_SCHEMA}},
|
|
514
|
+
"description": "snapshot, then progress events, then one end event (D-201)",
|
|
515
|
+
},
|
|
516
|
+
**_errors(404),
|
|
517
|
+
},
|
|
518
|
+
summary="Progress as server-sent events",
|
|
519
|
+
)
|
|
520
|
+
async def job_events(state: State, job_id: str) -> StreamingResponse:
|
|
521
|
+
job = _job(state, job_id)
|
|
522
|
+
|
|
523
|
+
async def stream() -> AsyncIterator[bytes]:
|
|
524
|
+
sent = job.version
|
|
525
|
+
snapshot = job.snapshot()
|
|
526
|
+
yield _event("snapshot", snapshot)
|
|
527
|
+
beat = time.monotonic()
|
|
528
|
+
while True:
|
|
529
|
+
if snapshot["state"] != JobState.RUNNING:
|
|
530
|
+
yield _event("end", snapshot)
|
|
531
|
+
return
|
|
532
|
+
if state.stopping.is_set():
|
|
533
|
+
return
|
|
534
|
+
await asyncio.sleep(POLL)
|
|
535
|
+
if job.version != sent:
|
|
536
|
+
sent = job.version
|
|
537
|
+
snapshot = job.snapshot()
|
|
538
|
+
if snapshot["state"] == JobState.RUNNING:
|
|
539
|
+
yield _event("progress", snapshot["progress"])
|
|
540
|
+
beat = time.monotonic()
|
|
541
|
+
elif time.monotonic() - beat >= KEEP_ALIVE:
|
|
542
|
+
yield b": keep-alive\n\n"
|
|
543
|
+
beat = time.monotonic()
|
|
544
|
+
|
|
545
|
+
return StreamingResponse(
|
|
546
|
+
stream(), media_type="text/event-stream", headers={"x-accel-buffering": "no"}
|
|
547
|
+
)
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
@router.get(
|
|
551
|
+
"/jobs/{job_id}/files/{name}",
|
|
552
|
+
response_class=FileResponse,
|
|
553
|
+
responses={**_FILE_ANSWERS, **_errors(404)},
|
|
554
|
+
summary="A file the job wrote",
|
|
555
|
+
)
|
|
556
|
+
def job_file(state: State, job_id: str, name: str, download: Download = False) -> FileResponse:
|
|
557
|
+
job = _job(state, job_id)
|
|
558
|
+
file = job.file(name)
|
|
559
|
+
if file is None or not file.path.is_file():
|
|
560
|
+
raise ApiError(404, "not-found", f"job {job_id} wrote no file {name!r}")
|
|
561
|
+
return _file(file.path, download)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _file(path: Path, download: bool) -> FileResponse:
|
|
565
|
+
"""A file as the browser shows it (`inline`) or saves it (`download`, D-255)."""
|
|
566
|
+
media_type = MEDIA_TYPES.get(path.suffix.lower(), "application/octet-stream")
|
|
567
|
+
disposition = "attachment" if download else "inline"
|
|
568
|
+
return FileResponse(
|
|
569
|
+
path, media_type=media_type, filename=path.name, content_disposition_type=disposition
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
# --- voices ----------------------------------------------------------------------------
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def plan_hosts(urls: list[str]) -> list[str]:
|
|
577
|
+
"""The hosts the voice download talks to, then where they redirect (D-141)."""
|
|
578
|
+
hosts = sorted({urlsplit(url).hostname or "" for url in urls} - {""})
|
|
579
|
+
return [*hosts, *(REDIRECTS[host] for host in hosts if host in REDIRECTS)]
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
@router.get(
|
|
583
|
+
"/voices",
|
|
584
|
+
response_model=models.VoicesOut,
|
|
585
|
+
responses=_errors(),
|
|
586
|
+
summary="Voices and the voice download",
|
|
587
|
+
)
|
|
588
|
+
def get_voices(state: State) -> models.VoicesOut:
|
|
589
|
+
plan = download_plan()
|
|
590
|
+
missing = missing_files(state.config.models_dir)
|
|
591
|
+
return models.VoicesOut(
|
|
592
|
+
voices=_voice_list(),
|
|
593
|
+
default_voice=DEFAULT_VOICE,
|
|
594
|
+
installed=not missing,
|
|
595
|
+
plan=models.PlanOut(
|
|
596
|
+
digest=plan_digest(plan),
|
|
597
|
+
total_bytes=plan.total_bytes,
|
|
598
|
+
missing_bytes=sum(file.size for file in missing),
|
|
599
|
+
licences=list(plan.licences),
|
|
600
|
+
hosts=plan_hosts([file.url for file in plan.files]),
|
|
601
|
+
files=[
|
|
602
|
+
models.PlanFileOut(
|
|
603
|
+
url=f.url, target=f.target, size=f.size, sha256=f.sha256, licence=f.licence
|
|
604
|
+
)
|
|
605
|
+
for f in plan.files
|
|
606
|
+
],
|
|
607
|
+
),
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
@router.post(
|
|
612
|
+
"/voices/download",
|
|
613
|
+
status_code=202,
|
|
614
|
+
response_model=models.JobOut,
|
|
615
|
+
responses=_errors(409, *BODY),
|
|
616
|
+
summary="Download the voice files, with the person's consent",
|
|
617
|
+
)
|
|
618
|
+
def download_voices(state: State, body: models.DownloadIn) -> models.JobOut:
|
|
619
|
+
if not body.consent:
|
|
620
|
+
raise ApiError(422, "no-consent", "the voice download needs the person's consent")
|
|
621
|
+
if body.plan != plan_digest(download_plan()):
|
|
622
|
+
raise ApiError(409, "stale-plan", "the download changed since it was shown; show it again")
|
|
623
|
+
work = voices_work(state.config.models_dir, state.fetch)
|
|
624
|
+
try:
|
|
625
|
+
job = state.board.start(JobKind.VOICES, None, None, work, cancellable=True)
|
|
626
|
+
except BoardBusyError as error:
|
|
627
|
+
raise ApiError(409, "busy", str(error)) from error
|
|
628
|
+
return _job_out(job)
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
@router.post(
|
|
632
|
+
"/voices/preview",
|
|
633
|
+
status_code=202,
|
|
634
|
+
response_model=models.JobOut,
|
|
635
|
+
responses=_errors(409, *BODY),
|
|
636
|
+
summary="Speak a preview sentence with one voice at one speed",
|
|
637
|
+
)
|
|
638
|
+
def preview_voice(state: State, body: models.PreviewIn) -> models.JobOut:
|
|
639
|
+
"""A preview job (D-249): its one file is the WAV to play. The voice stays loaded
|
|
640
|
+
afterwards for the next preview and the audio job (D-250)."""
|
|
641
|
+
text = " ".join(body.text.split())
|
|
642
|
+
if not text:
|
|
643
|
+
raise ApiError(422, "invalid", "text: the preview sentence is empty")
|
|
644
|
+
settings = _job_settings(voice=_voice(body.voice), speed=body.speed)
|
|
645
|
+
if missing_files(state.config.models_dir):
|
|
646
|
+
raise ApiError(409, "voice-missing", "the voice is not installed yet; download it first")
|
|
647
|
+
work = state.preview_work(text, settings.voice, settings.speed)
|
|
648
|
+
try:
|
|
649
|
+
job = state.board.start(
|
|
650
|
+
JobKind.PREVIEW,
|
|
651
|
+
None,
|
|
652
|
+
{"voice": settings.voice, "speed": settings.speed},
|
|
653
|
+
work,
|
|
654
|
+
cancellable=False,
|
|
655
|
+
)
|
|
656
|
+
except BoardBusyError as error:
|
|
657
|
+
raise ApiError(409, "busy", str(error)) from error
|
|
658
|
+
return _job_out(job)
|