sci2audio 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sci2audio/__init__.py +3 -0
- sci2audio/app/__init__.py +110 -0
- sci2audio/app/audio.py +847 -0
- sci2audio/app/chapters.py +109 -0
- sci2audio/app/chunks.py +102 -0
- sci2audio/app/config.py +136 -0
- sci2audio/app/estimate.py +192 -0
- sci2audio/app/files.py +98 -0
- sci2audio/app/job.py +285 -0
- sci2audio/app/lock.py +92 -0
- sci2audio/app/report.py +400 -0
- sci2audio/app/runner.py +92 -0
- sci2audio/app/script.py +95 -0
- sci2audio/app/settings.py +58 -0
- sci2audio/app/voice.py +70 -0
- sci2audio/cli.py +1013 -0
- sci2audio/core/__init__.py +81 -0
- sci2audio/core/audio.py +39 -0
- sci2audio/core/content.py +266 -0
- sci2audio/core/document.py +225 -0
- sci2audio/core/edits.py +121 -0
- sci2audio/core/export.py +76 -0
- sci2audio/extraction/__init__.py +28 -0
- sci2audio/extraction/cover.py +68 -0
- sci2audio/extraction/epub.py +576 -0
- sci2audio/extraction/errors.py +7 -0
- sci2audio/extraction/markdown.py +250 -0
- sci2audio/extraction/markup.py +356 -0
- sci2audio/extraction/pdf.py +623 -0
- sci2audio/extraction/text.py +136 -0
- sci2audio/layout/__init__.py +13 -0
- sci2audio/layout/build.py +265 -0
- sci2audio/layout/captions.py +77 -0
- sci2audio/layout/footnotes.py +180 -0
- sci2audio/layout/formulas.py +392 -0
- sci2audio/layout/furniture.py +326 -0
- sci2audio/layout/geometry.py +124 -0
- sci2audio/layout/graphics.py +569 -0
- sci2audio/layout/headings.py +358 -0
- sci2audio/layout/lists.py +167 -0
- sci2audio/layout/maths.py +47 -0
- sci2audio/layout/order.py +374 -0
- sci2audio/layout/paragraphs.py +518 -0
- sci2audio/layout/settings.py +142 -0
- sci2audio/layout/split.py +113 -0
- sci2audio/layout/tagged.py +442 -0
- sci2audio/output/__init__.py +44 -0
- sci2audio/output/ffmpeg.py +111 -0
- sci2audio/output/model.py +125 -0
- sci2audio/output/names.py +70 -0
- sci2audio/output/readalong.py +160 -0
- sci2audio/output/timeline.py +113 -0
- sci2audio/output/wav.py +83 -0
- sci2audio/output/write.py +280 -0
- sci2audio/testing/__init__.py +32 -0
- sci2audio/testing/documents.py +124 -0
- sci2audio/testing/regression.py +197 -0
- sci2audio/testing/text_steps.py +262 -0
- sci2audio/testing/voices.py +164 -0
- sci2audio/textprep/__init__.py +105 -0
- sci2audio/textprep/abbreviations.py +138 -0
- sci2audio/textprep/characters.py +267 -0
- sci2audio/textprep/citations.py +683 -0
- sci2audio/textprep/data/en/abbreviations.toml +209 -0
- sci2audio/textprep/data/en/units.toml +548 -0
- sci2audio/textprep/footnotes.py +418 -0
- sci2audio/textprep/furniture.py +46 -0
- sci2audio/textprep/hyphens.py +228 -0
- sci2audio/textprep/links.py +628 -0
- sci2audio/textprep/notation.py +170 -0
- sci2audio/textprep/numbers.py +529 -0
- sci2audio/textprep/placeholders.py +68 -0
- sci2audio/textprep/references.py +107 -0
- sci2audio/textprep/registry.py +83 -0
- sci2audio/textprep/runner.py +206 -0
- sci2audio/textprep/spoken.py +33 -0
- sci2audio/textprep/statistics.py +322 -0
- sci2audio/textprep/steps.py +153 -0
- sci2audio/textprep/units.py +449 -0
- sci2audio/textprep/utterances.py +315 -0
- sci2audio/textprep/wording.py +93 -0
- sci2audio/textprep/wordlists.py +189 -0
- sci2audio/voices/__init__.py +102 -0
- sci2audio/voices/download.py +340 -0
- sci2audio/voices/engine.py +136 -0
- sci2audio/voices/errors.py +88 -0
- sci2audio/voices/kokoro_engine.py +346 -0
- sci2audio/voices/runtime.py +37 -0
- sci2audio/voices/types.py +206 -0
- sci2audio/voices/wav.py +82 -0
- sci2audio/web/__init__.py +13 -0
- sci2audio/web/api.py +658 -0
- sci2audio/web/config.py +97 -0
- sci2audio/web/documents.py +496 -0
- sci2audio/web/i18n.py +188 -0
- sci2audio/web/jobs.py +265 -0
- sci2audio/web/launch.py +121 -0
- sci2audio/web/limits.py +64 -0
- sci2audio/web/models.py +178 -0
- sci2audio/web/pages.py +509 -0
- sci2audio/web/player.py +549 -0
- sci2audio/web/security.py +194 -0
- sci2audio/web/server.py +116 -0
- sci2audio/web/state.py +88 -0
- sci2audio/web/static/app.css +150 -0
- sci2audio/web/static/app.js +499 -0
- sci2audio/web/static/player.js +277 -0
- sci2audio/web/strings/de.toml +348 -0
- sci2audio/web/strings/en.toml +343 -0
- sci2audio/web/tasks.py +280 -0
- sci2audio/web/templates/_remove.html +17 -0
- sci2audio/web/templates/base.html +32 -0
- sci2audio/web/templates/document.html +153 -0
- sci2audio/web/templates/index.html +41 -0
- sci2audio/web/templates/job.html +66 -0
- sci2audio/web/templates/listen.html +75 -0
- sci2audio/web/templates/missing.html +7 -0
- sci2audio/web/templates/privacy.html +47 -0
- sci2audio/web/templates/voice.html +31 -0
- sci2audio-1.0.0.dist-info/METADATA +171 -0
- sci2audio-1.0.0.dist-info/RECORD +125 -0
- sci2audio-1.0.0.dist-info/WHEEL +4 -0
- sci2audio-1.0.0.dist-info/entry_points.txt +2 -0
- sci2audio-1.0.0.dist-info/licenses/LICENSE +674 -0
- sci2audio-1.0.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +86 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Chapters: what the person chooses to hear, and what output turns into chapter marks (Q6).
|
|
2
|
+
|
|
3
|
+
- Each top-level section of the document starts a chapter (a level-1 section; a top-level
|
|
4
|
+
section of a deeper level, with no level-1 heading above it, too). A chapter runs to the
|
|
5
|
+
next chapter's start or the document's end, so every block is in exactly one chapter,
|
|
6
|
+
also where the section tree leaves gaps.
|
|
7
|
+
- Text before the first heading is the first chapter, titled with the document title the
|
|
8
|
+
file gives (`Content.metadata.title`), else the file name without its extension. A
|
|
9
|
+
document without headings is one chapter, titled the same way.
|
|
10
|
+
- A chapter whose title, without its printed number, is a reference-list name
|
|
11
|
+
(`REFERENCE_NAMES`) is the reference list: deselected by default (TP-06). Appendices stay
|
|
12
|
+
selected.
|
|
13
|
+
|
|
14
|
+
Titles are kept as printed ("2 Related Work"), white space runs read as one space.
|
|
15
|
+
EPUB's top-level navigation entries become level-1 sections in layout (#73, #75).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import re
|
|
21
|
+
from collections.abc import Collection
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from pathlib import PurePath
|
|
24
|
+
|
|
25
|
+
from sci2audio.core import Document
|
|
26
|
+
from sci2audio.textprep import REFERENCE_NAMES
|
|
27
|
+
|
|
28
|
+
# The names a reference-list heading has, compared without case and without a printed
|
|
29
|
+
# number: the reference-list text step's own table, English and German (#221), so the
|
|
30
|
+
# chapter choice and the step agree (D-146). Every language's names count here, since the
|
|
31
|
+
# chapter list does not depend on the job's language.
|
|
32
|
+
_ALL_REFERENCE_NAMES = frozenset(
|
|
33
|
+
name.casefold() for names in REFERENCE_NAMES.values() for name in names
|
|
34
|
+
)
|
|
35
|
+
# a printed heading number before the title: "7", "7.", "A", "A.1", "IV."
|
|
36
|
+
_NUMBER = re.compile(r"\A(?:\d{1,2}|[A-Z]|[IVX]{1,5})(?:\.\d{1,2})*\.?\s+")
|
|
37
|
+
_SPACES = re.compile(r"\s+")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Chapter:
|
|
42
|
+
"""Chapter number `index` (from 0): its title and blocks `start` up to `end`."""
|
|
43
|
+
|
|
44
|
+
index: int
|
|
45
|
+
title: str
|
|
46
|
+
start: int
|
|
47
|
+
end: int
|
|
48
|
+
references: bool = False # the reference list: deselected by default
|
|
49
|
+
|
|
50
|
+
def __post_init__(self) -> None:
|
|
51
|
+
if not self.title.strip():
|
|
52
|
+
raise ValueError("a chapter needs a title")
|
|
53
|
+
if self.index < 0 or not 0 <= self.start <= self.end:
|
|
54
|
+
raise ValueError(f"chapter {self.title!r} needs index >= 0 and start <= end")
|
|
55
|
+
|
|
56
|
+
def holds(self, block: int) -> bool:
|
|
57
|
+
"""True if block number `block` belongs to this chapter."""
|
|
58
|
+
return self.start <= block < self.end
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def document_title(document: Document) -> str:
|
|
62
|
+
"""The title the file gives, else the file name without its extension ("Untitled" if
|
|
63
|
+
neither has a letter to read)."""
|
|
64
|
+
name = document.content.source.name
|
|
65
|
+
candidates = (document.content.metadata.title, PurePath(name).stem, name)
|
|
66
|
+
return next((title for text in candidates if (title := _clean(text))), "Untitled")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def find_chapters(document: Document) -> tuple[Chapter, ...]:
|
|
70
|
+
"""The document's chapters in order; together they hold every block exactly once."""
|
|
71
|
+
count = len(document.blocks)
|
|
72
|
+
starts = [(section.start, section.title) for section in document.sections]
|
|
73
|
+
heads: list[tuple[int, str, bool]] = [] # start, title, reference list
|
|
74
|
+
if not starts or starts[0][0] > 0:
|
|
75
|
+
heads.append((0, document_title(document), False))
|
|
76
|
+
for number, (start, title) in enumerate(starts, start=1):
|
|
77
|
+
cleaned = _clean(title) or f"Section {number}"
|
|
78
|
+
heads.append((start, cleaned, is_reference_title(cleaned)))
|
|
79
|
+
chapters = []
|
|
80
|
+
for index, (start, title, references) in enumerate(heads):
|
|
81
|
+
end = heads[index + 1][0] if index + 1 < len(heads) else count
|
|
82
|
+
chapters.append(Chapter(index, title, start, end, references))
|
|
83
|
+
return tuple(chapters)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def is_reference_title(title: str) -> bool:
|
|
87
|
+
"""True if the title names a reference list ("References", "7 Bibliography")."""
|
|
88
|
+
name = _NUMBER.sub("", _clean(title), count=1)
|
|
89
|
+
return name.casefold().rstrip(".:") in _ALL_REFERENCE_NAMES
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def default_selection(chapters: Collection[Chapter]) -> frozenset[int]:
|
|
93
|
+
"""Every chapter but the reference list."""
|
|
94
|
+
return frozenset(chapter.index for chapter in chapters if not chapter.references)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def selected(
|
|
98
|
+
chapters: Collection[Chapter], chosen: Collection[int] | None
|
|
99
|
+
) -> tuple[frozenset[int], tuple[int, ...]]:
|
|
100
|
+
"""The chosen chapters (the default when `chosen` is None), and the chosen indices the
|
|
101
|
+
document doesn't have, which are ignored."""
|
|
102
|
+
if chosen is None:
|
|
103
|
+
return default_selection(chapters), ()
|
|
104
|
+
known = {chapter.index for chapter in chapters}
|
|
105
|
+
return frozenset(chosen) & known, tuple(sorted(set(chosen) - known))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _clean(text: str) -> str:
|
|
109
|
+
return _SPACES.sub(" ", text).strip()
|
sci2audio/app/chunks.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""The audio job's chunk cache in the working folder (WI-02, D-155, D-156, D-158).
|
|
2
|
+
|
|
3
|
+
Every chunk is the voice's audio of one utterance, cached under a key of what decides the
|
|
4
|
+
sound: the text, the voice, the speed and the engine version (the span is not part of it,
|
|
5
|
+
so the same text twice in a job is spoken once). A chunk is finished when its WAV file is
|
|
6
|
+
complete: the voice writes `<name>.part` and renames it (`voices.write_wav`), so a run
|
|
7
|
+
killed outright leaves at most a `.part` file, which the next run removes. Resuming is
|
|
8
|
+
running the job again on the same working folder: every finished chunk is used again.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import contextlib
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import shutil
|
|
17
|
+
import wave
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from sci2audio.app.lock import work_folder_lock
|
|
21
|
+
|
|
22
|
+
CHUNKS = "chunks" # the cache's folder inside the working folder
|
|
23
|
+
KEY_FORMAT = 1 # bump when anything else starts to change the sound
|
|
24
|
+
_SAMPLE_WIDTH = 2 # 16-bit PCM (D-117)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def chunk_key(text: str, voice: str, speed: float, engine: str, engine_version: str) -> str:
|
|
28
|
+
"""The cache key of a chunk: SHA-256 (hex) of a canonical JSON of what decides its sound."""
|
|
29
|
+
data = {
|
|
30
|
+
"format": KEY_FORMAT,
|
|
31
|
+
"engine": engine,
|
|
32
|
+
"engine_version": engine_version,
|
|
33
|
+
"voice": voice,
|
|
34
|
+
"speed": float(speed),
|
|
35
|
+
"text": text,
|
|
36
|
+
}
|
|
37
|
+
canonical = json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
38
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def chunk_file(key: str) -> str:
|
|
42
|
+
"""The chunk's file relative to the working folder (an `AudioChunk.file`)."""
|
|
43
|
+
return f"{CHUNKS}/{key[:2]}/{key}.wav"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def is_finished(path: Path) -> bool:
|
|
47
|
+
"""True if `path` is a complete 16-bit mono PCM WAV file (a finished chunk)."""
|
|
48
|
+
try:
|
|
49
|
+
with wave.open(str(path), "rb") as audio:
|
|
50
|
+
if audio.getnchannels() != 1 or audio.getsampwidth() != _SAMPLE_WIDTH:
|
|
51
|
+
return False
|
|
52
|
+
if audio.getframerate() <= 0:
|
|
53
|
+
return False
|
|
54
|
+
frames = audio.getnframes()
|
|
55
|
+
if frames == 0:
|
|
56
|
+
return True
|
|
57
|
+
audio.setpos(frames - 1) # a file cut short has fewer bytes than its header says
|
|
58
|
+
return len(audio.readframes(1)) == _SAMPLE_WIDTH
|
|
59
|
+
except (OSError, EOFError, wave.Error):
|
|
60
|
+
return False
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def remove_partials(work: Path) -> int:
|
|
64
|
+
"""Remove the `.part` files a killed run left in the cache; return how many."""
|
|
65
|
+
removed = 0
|
|
66
|
+
folder = work / CHUNKS
|
|
67
|
+
if not folder.is_dir():
|
|
68
|
+
return 0
|
|
69
|
+
for part in folder.glob("*/*.part"):
|
|
70
|
+
with contextlib.suppress(OSError):
|
|
71
|
+
part.unlink()
|
|
72
|
+
removed += 1
|
|
73
|
+
return removed
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def cache_size(work: Path) -> int:
|
|
77
|
+
"""The bytes the chunk cache takes (0 when there is none)."""
|
|
78
|
+
folder = work / CHUNKS
|
|
79
|
+
if not folder.is_dir():
|
|
80
|
+
return 0
|
|
81
|
+
total = 0
|
|
82
|
+
for path in folder.rglob("*"):
|
|
83
|
+
with contextlib.suppress(OSError):
|
|
84
|
+
if path.is_file():
|
|
85
|
+
total += path.stat().st_size
|
|
86
|
+
return total
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def clean_work_folder(work: Path) -> int:
|
|
90
|
+
"""Remove the chunk cache from a working folder; return the bytes freed.
|
|
91
|
+
|
|
92
|
+
Only the `chunks` folder goes; the records (`audio.json`, `document.json`, `job.json`)
|
|
93
|
+
and the lock file stay, so a finished job can still be listed and played. The next run
|
|
94
|
+
speaks again. It holds the working folder's lock while it removes the cache, so it never
|
|
95
|
+
removes chunks under a running job: JobBusyError while a job works on the folder (D-226).
|
|
96
|
+
"""
|
|
97
|
+
if not work.is_dir():
|
|
98
|
+
return 0 # nothing to clean; the lock would only create the folder
|
|
99
|
+
with work_folder_lock(work):
|
|
100
|
+
size = cache_size(work)
|
|
101
|
+
shutil.rmtree(work / CHUNKS, ignore_errors=True)
|
|
102
|
+
return size
|
sci2audio/app/config.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""The settings file: the person's choices kept between runs (playbook 3.4).
|
|
2
|
+
|
|
3
|
+
A JSON file in the user's configuration folder (`default_config_file`, from platformdirs).
|
|
4
|
+
Only the command line and the server read and write it; a job gets its settings passed in
|
|
5
|
+
and never touches the file. The chosen chapters are not kept: they belong to one document.
|
|
6
|
+
A missing, unreadable or broken file never stops the program: what can't be read falls
|
|
7
|
+
back to the default, with a warning the caller shows.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import platformdirs
|
|
18
|
+
|
|
19
|
+
from sci2audio.app.files import write_text_atomically
|
|
20
|
+
from sci2audio.app.settings import JobSettings
|
|
21
|
+
from sci2audio.textprep import Category
|
|
22
|
+
|
|
23
|
+
CONFIG_NAME = "settings.json"
|
|
24
|
+
CONFIG_VERSION = 1
|
|
25
|
+
KEPT = ( # every JobSettings field but the chosen chapters
|
|
26
|
+
"language",
|
|
27
|
+
"switched_off",
|
|
28
|
+
"voice",
|
|
29
|
+
"speed",
|
|
30
|
+
"output_folder",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def default_config_file() -> Path:
|
|
35
|
+
"""Where the settings file lives: the user's configuration folder for sci2audio.
|
|
36
|
+
|
|
37
|
+
Windows `%APPDATA%\\sci2audio`, macOS `~/Library/Application Support/sci2audio`,
|
|
38
|
+
Linux `$XDG_CONFIG_HOME/sci2audio` (usually `~/.config/sci2audio`).
|
|
39
|
+
"""
|
|
40
|
+
folder = platformdirs.user_config_path("sci2audio", appauthor=False, roaming=True)
|
|
41
|
+
return folder / CONFIG_NAME
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def default_models_folder() -> Path:
|
|
45
|
+
"""Where the voice setup puts the voice files (D-164): the user's data folder, not the
|
|
46
|
+
cache, so a system clean-up never deletes the 355 MB download."""
|
|
47
|
+
return platformdirs.user_data_path("sci2audio", appauthor=False) / "models"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def default_work_folder(source: Path) -> Path:
|
|
51
|
+
"""The working folder of an audio job (D-164): one per source file, named by the first
|
|
52
|
+
16 hex digits of the file's SHA-256, in the user's cache folder. The same file under
|
|
53
|
+
another name or in another folder gets the same folder, so its chunks are used again.
|
|
54
|
+
Raises OSError if the file can't be read."""
|
|
55
|
+
digest = hashlib.sha256()
|
|
56
|
+
with source.open("rb") as file:
|
|
57
|
+
for block in iter(lambda: file.read(1 << 20), b""):
|
|
58
|
+
digest.update(block)
|
|
59
|
+
folder = platformdirs.user_cache_path("sci2audio", appauthor=False)
|
|
60
|
+
return folder / "jobs" / digest.hexdigest()[:16]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def settings_to_data(settings: JobSettings) -> dict[str, Any]:
|
|
64
|
+
"""The settings as the file keeps them (without the chosen chapters)."""
|
|
65
|
+
return {
|
|
66
|
+
"version": CONFIG_VERSION,
|
|
67
|
+
"language": settings.language,
|
|
68
|
+
"switched_off": sorted(str(category) for category in settings.switched_off),
|
|
69
|
+
"voice": settings.voice,
|
|
70
|
+
"speed": settings.speed,
|
|
71
|
+
"output_folder": None if settings.output_folder is None else str(settings.output_folder),
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def settings_from_data(data: object) -> tuple[JobSettings, list[str]]:
|
|
76
|
+
"""Settings from the file's data, and a warning for everything that was left out."""
|
|
77
|
+
if not isinstance(data, dict):
|
|
78
|
+
return JobSettings(), ["the settings file does not hold settings; using the defaults"]
|
|
79
|
+
warnings = []
|
|
80
|
+
version = data.get("version")
|
|
81
|
+
if type(version) is not int or version != CONFIG_VERSION: # true is not version 1
|
|
82
|
+
warnings.append(f"the settings file has version {version!r}; reading what fits")
|
|
83
|
+
values: dict[str, Any] = {}
|
|
84
|
+
for key, value in data.items():
|
|
85
|
+
if key == "version":
|
|
86
|
+
continue
|
|
87
|
+
if key not in KEPT:
|
|
88
|
+
warnings.append(f"unknown setting {key!r} ignored")
|
|
89
|
+
continue
|
|
90
|
+
try:
|
|
91
|
+
converted = _convert(key, value)
|
|
92
|
+
JobSettings(**{key: converted}) # check it on its own
|
|
93
|
+
except (TypeError, ValueError) as error:
|
|
94
|
+
warnings.append(f"setting {key!r} ignored: {error}")
|
|
95
|
+
else:
|
|
96
|
+
values[key] = converted
|
|
97
|
+
return JobSettings(**values), warnings
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def load_settings(file: Path) -> tuple[JobSettings, list[str]]:
|
|
101
|
+
"""The settings kept in `file`, and warnings; the defaults if there is no file."""
|
|
102
|
+
try:
|
|
103
|
+
text = file.read_text(encoding="utf-8")
|
|
104
|
+
except FileNotFoundError:
|
|
105
|
+
return JobSettings(), []
|
|
106
|
+
except (OSError, UnicodeDecodeError) as error:
|
|
107
|
+
return JobSettings(), [f"the settings file can't be read ({error}); using the defaults"]
|
|
108
|
+
try:
|
|
109
|
+
data = json.loads(text)
|
|
110
|
+
except (ValueError, RecursionError) as error: # JSONDecodeError; nesting too deep
|
|
111
|
+
detail = error if isinstance(error, ValueError) else "nested too deeply"
|
|
112
|
+
return JobSettings(), [f"the settings file is damaged ({detail}); using the defaults"]
|
|
113
|
+
return settings_from_data(data)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def save_settings(settings: JobSettings, file: Path) -> None:
|
|
117
|
+
"""Keep the settings in `file`, all or nothing (OSError if it can't be written)."""
|
|
118
|
+
file.parent.mkdir(parents=True, exist_ok=True)
|
|
119
|
+
text = json.dumps(settings_to_data(settings), ensure_ascii=False, indent=2) + "\n"
|
|
120
|
+
write_text_atomically(file, text, overwrite=True)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _convert(key: str, value: object) -> object:
|
|
124
|
+
if key == "switched_off":
|
|
125
|
+
if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
|
|
126
|
+
raise ValueError(f"expected a list of category names, got {value!r}")
|
|
127
|
+
return frozenset(Category(name) for name in value)
|
|
128
|
+
if key == "output_folder":
|
|
129
|
+
if value is None:
|
|
130
|
+
return None
|
|
131
|
+
if not isinstance(value, str) or not value:
|
|
132
|
+
raise ValueError(f"expected a folder, got {value!r}")
|
|
133
|
+
return Path(value)
|
|
134
|
+
if key in ("language", "voice") and not isinstance(value, str):
|
|
135
|
+
raise ValueError(f"expected text, got {value!r}")
|
|
136
|
+
return value
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""The audio job's time estimate: what one voice call costs, fitted on this run (#136).
|
|
2
|
+
|
|
3
|
+
A voice call takes a fixed time on top of its characters, and long utterances take more
|
|
4
|
+
per character than short ones, so the seconds of one call are modelled as
|
|
5
|
+
|
|
6
|
+
seconds = constant + per_character * c + per_square * c * c (c = characters)
|
|
7
|
+
|
|
8
|
+
with every coefficient zero or more, fitted by least squares on the chunks measured so far
|
|
9
|
+
in this run (D-380). The time left is that cost summed over every chunk still to speak.
|
|
10
|
+
|
|
11
|
+
What keeps a curve from running wild:
|
|
12
|
+
- fewer than MIN_CHUNKS measured chunks (twice the coefficients), or fewer different
|
|
13
|
+
lengths than coefficients: the plain rate (measured seconds / measured characters,
|
|
14
|
+
D-159) instead of the fit;
|
|
15
|
+
- the curve is never used beyond the longest measured chunk L: a longer chunk costs
|
|
16
|
+
model(L) / L per character, the cost per character at the edge of what was measured
|
|
17
|
+
(D-384). A first chapter of names, headings or a list measures only short chunks,
|
|
18
|
+
and the curve fitted on them can say almost twice the real time for long paragraphs.
|
|
19
|
+
The fit also keeps the plain rate when it explains the measured chunks no better than the
|
|
20
|
+
rate does; since the per-character-only fit is one of its candidates, that happens only
|
|
21
|
+
when the cost is exactly per character.
|
|
22
|
+
|
|
23
|
+
The measured chunks are kept as running sums, so adding one is constant time; the chunks
|
|
24
|
+
still to speak are counted by length, so the time left costs one step per different
|
|
25
|
+
length (at most the voice's longest utterance, 300 characters by default).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import itertools
|
|
31
|
+
from collections import Counter
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
|
|
34
|
+
TERMS = 3 # constant, per character, per character squared
|
|
35
|
+
MIN_CHUNKS = 2 * TERMS # measured chunks before the fit is trusted over the plain rate
|
|
36
|
+
SCALE = 100.0 # characters are fitted in hundreds, so the sums stay well conditioned
|
|
37
|
+
_PIVOT = 1e-9 # a smaller pivot (relative to the diagonal) means the terms cannot be told apart
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class CostModel:
|
|
42
|
+
"""Seconds one voice call takes for `characters` characters."""
|
|
43
|
+
|
|
44
|
+
constant: float = 0.0
|
|
45
|
+
per_character: float = 0.0
|
|
46
|
+
per_square: float = 0.0
|
|
47
|
+
|
|
48
|
+
def seconds(self, characters: int) -> float:
|
|
49
|
+
c = float(characters)
|
|
50
|
+
return self.constant + self.per_character * c + self.per_square * c * c
|
|
51
|
+
|
|
52
|
+
@classmethod
|
|
53
|
+
def rate(cls, seconds: float, characters: int) -> CostModel:
|
|
54
|
+
"""The plain rate: every character costs the same, nothing per call."""
|
|
55
|
+
return cls(per_character=seconds / characters)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class _Sums:
|
|
60
|
+
"""Running sums of x**k (k = 0..2*TERMS-2) and y*x**k (k = 0..TERMS-1) and y*y."""
|
|
61
|
+
|
|
62
|
+
powers: list[float] = field(default_factory=lambda: [0.0] * (2 * TERMS - 1))
|
|
63
|
+
weighted: list[float] = field(default_factory=lambda: [0.0] * TERMS)
|
|
64
|
+
squares: float = 0.0
|
|
65
|
+
|
|
66
|
+
def add(self, x: float, y: float) -> None:
|
|
67
|
+
for k in range(len(self.powers)):
|
|
68
|
+
self.powers[k] += x**k
|
|
69
|
+
for k in range(TERMS):
|
|
70
|
+
self.weighted[k] += y * x**k
|
|
71
|
+
self.squares += y * y
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class Estimate:
|
|
76
|
+
"""The measured chunks of one run and the chunks still to speak."""
|
|
77
|
+
|
|
78
|
+
measured_characters: int = 0
|
|
79
|
+
measured_seconds: float = 0.0
|
|
80
|
+
measured_chunks: int = 0
|
|
81
|
+
_measured: _Sums = field(default_factory=_Sums)
|
|
82
|
+
_lengths: set[int] = field(default_factory=set)
|
|
83
|
+
_left: Counter[int] = field(default_factory=Counter) # length -> chunks still to speak
|
|
84
|
+
|
|
85
|
+
def measure(self, characters: int, seconds: float) -> None:
|
|
86
|
+
"""One voice call of `characters` characters took `seconds`."""
|
|
87
|
+
self.measured_characters += characters
|
|
88
|
+
self.measured_seconds += seconds
|
|
89
|
+
self.measured_chunks += 1
|
|
90
|
+
self._measured.add(characters / SCALE, seconds)
|
|
91
|
+
self._lengths.add(characters)
|
|
92
|
+
|
|
93
|
+
def expect(self, characters: int) -> None:
|
|
94
|
+
"""A chunk of `characters` characters is still to be spoken."""
|
|
95
|
+
self._left[characters] += 1
|
|
96
|
+
|
|
97
|
+
def done(self, characters: int) -> None:
|
|
98
|
+
"""A chunk `expect`ed earlier is spoken (or failed): it is no longer left."""
|
|
99
|
+
self._left[characters] -= 1
|
|
100
|
+
if self._left[characters] <= 0:
|
|
101
|
+
del self._left[characters]
|
|
102
|
+
|
|
103
|
+
def speed(self) -> float | None:
|
|
104
|
+
"""Measured characters per second, the plain rate; None before any measurement."""
|
|
105
|
+
if self.measured_characters == 0 or self.measured_seconds <= 0:
|
|
106
|
+
return None
|
|
107
|
+
return self.measured_characters / self.measured_seconds
|
|
108
|
+
|
|
109
|
+
def model(self) -> CostModel | None:
|
|
110
|
+
"""The cost of one call, fitted on the measured chunks; None before any."""
|
|
111
|
+
if self.speed() is None:
|
|
112
|
+
return None
|
|
113
|
+
rate = CostModel.rate(self.measured_seconds, self.measured_characters)
|
|
114
|
+
if self.measured_chunks < MIN_CHUNKS or len(self._lengths) < TERMS:
|
|
115
|
+
return rate
|
|
116
|
+
fitted = _fit(self._measured)
|
|
117
|
+
if fitted is None:
|
|
118
|
+
return rate
|
|
119
|
+
coefficients, error = fitted
|
|
120
|
+
if error >= self._error(rate):
|
|
121
|
+
return rate
|
|
122
|
+
constant, per_character, per_square = coefficients
|
|
123
|
+
return CostModel(constant, per_character / SCALE, per_square / SCALE**2)
|
|
124
|
+
|
|
125
|
+
def seconds_left(self) -> float | None:
|
|
126
|
+
"""The model summed over every chunk still to speak, never beyond the longest
|
|
127
|
+
measured chunk; None before any measurement."""
|
|
128
|
+
model = self.model()
|
|
129
|
+
if model is None:
|
|
130
|
+
return None
|
|
131
|
+
longest = max(self._lengths)
|
|
132
|
+
per_character_beyond = model.seconds(longest) / longest
|
|
133
|
+
seconds = sum(
|
|
134
|
+
count * (model.seconds(length) if length <= longest else per_character_beyond * length)
|
|
135
|
+
for length, count in self._left.items()
|
|
136
|
+
)
|
|
137
|
+
return max(0.0, seconds)
|
|
138
|
+
|
|
139
|
+
def _error(self, model: CostModel) -> float:
|
|
140
|
+
"""Sum of squared errors of `model` over the measured chunks."""
|
|
141
|
+
p = (model.constant, model.per_character * SCALE, model.per_square * SCALE**2)
|
|
142
|
+
return _sse(self._measured, p)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _sse(sums: _Sums, p: tuple[float, ...]) -> float:
|
|
146
|
+
"""Sum over the measured chunks of (y - sum p_k x**k)**2, from the running sums."""
|
|
147
|
+
cross = sum(p[k] * sums.weighted[k] for k in range(TERMS))
|
|
148
|
+
quad = sum(p[i] * p[j] * sums.powers[i + j] for i in range(TERMS) for j in range(TERMS))
|
|
149
|
+
return max(0.0, sums.squares - 2 * cross + quad)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _fit(sums: _Sums) -> tuple[tuple[float, ...], float] | None:
|
|
153
|
+
"""Non-negative least squares over the terms, exactly: the best unconstrained fit of
|
|
154
|
+
every subset of terms whose coefficients all come out zero or more. None if no subset
|
|
155
|
+
can be solved."""
|
|
156
|
+
best: tuple[tuple[float, ...], float] | None = None
|
|
157
|
+
for size in range(1, TERMS + 1):
|
|
158
|
+
for subset in itertools.combinations(range(TERMS), size):
|
|
159
|
+
solved = _solve(
|
|
160
|
+
[[sums.powers[i + j] for j in subset] for i in subset],
|
|
161
|
+
[sums.weighted[i] for i in subset],
|
|
162
|
+
)
|
|
163
|
+
if solved is None or any(value < 0 for value in solved):
|
|
164
|
+
continue
|
|
165
|
+
p = [0.0] * TERMS
|
|
166
|
+
for index, value in zip(subset, solved, strict=True):
|
|
167
|
+
p[index] = value
|
|
168
|
+
error = _sse(sums, tuple(p))
|
|
169
|
+
if best is None or error < best[1]:
|
|
170
|
+
best = (tuple(p), error)
|
|
171
|
+
return best
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _solve(matrix: list[list[float]], vector: list[float]) -> list[float] | None:
|
|
175
|
+
"""Gaussian elimination with partial pivoting; None if the matrix is (nearly) singular."""
|
|
176
|
+
n = len(vector)
|
|
177
|
+
rows = [[*matrix[i], vector[i]] for i in range(n)]
|
|
178
|
+
scale = max(abs(rows[i][i]) for i in range(n)) or 1.0
|
|
179
|
+
for column in range(n):
|
|
180
|
+
pivot = max(range(column, n), key=lambda r: abs(rows[r][column]))
|
|
181
|
+
if abs(rows[pivot][column]) <= _PIVOT * scale:
|
|
182
|
+
return None
|
|
183
|
+
rows[column], rows[pivot] = rows[pivot], rows[column]
|
|
184
|
+
for r in range(column + 1, n):
|
|
185
|
+
factor = rows[r][column] / rows[column][column]
|
|
186
|
+
for k in range(column, n + 1):
|
|
187
|
+
rows[r][k] -= factor * rows[column][k]
|
|
188
|
+
solution = [0.0] * n
|
|
189
|
+
for r in reversed(range(n)):
|
|
190
|
+
total = rows[r][n] - sum(rows[r][k] * solution[k] for k in range(r + 1, n))
|
|
191
|
+
solution[r] = total / rows[r][r]
|
|
192
|
+
return solution
|
sci2audio/app/files.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Writing the job's files so that no half-written file is ever left (the #69 rule).
|
|
2
|
+
|
|
3
|
+
Each file is written under a temporary name (`.<name>.partial`) next to its target and
|
|
4
|
+
renamed when complete; on any error the partial files are removed. `write_texts` writes
|
|
5
|
+
all its files before it renames any, so a full disk leaves every earlier file as it was,
|
|
6
|
+
and without `overwrite` it puts either all of its files in place or none.
|
|
7
|
+
An existing file is only replaced with `overwrite=True`; without it, a file that appeared
|
|
8
|
+
at the target during the run is never replaced either. A run killed outright may leave a
|
|
9
|
+
partial file behind; the next run writes over it.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import contextlib
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
from collections.abc import Mapping
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def partial_path(target: Path) -> Path:
|
|
22
|
+
"""The temporary name a file is written under."""
|
|
23
|
+
return target.with_name(f".{target.name}.partial")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def write_text_atomically(target: Path, text: str, *, overwrite: bool) -> None:
|
|
27
|
+
"""Write `text` (UTF-8, "\\n" line ends) to `target`, all of it or nothing."""
|
|
28
|
+
write_texts({target: text}, overwrite=overwrite)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def write_bytes_atomically(target: Path, data: bytes, *, overwrite: bool) -> None:
|
|
32
|
+
"""Write `data` to `target`, all of it or nothing (as `write_texts`)."""
|
|
33
|
+
_write_all({target: data}, overwrite=overwrite)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def write_texts(texts: Mapping[Path, str], *, overwrite: bool) -> None:
|
|
37
|
+
"""Write each text to its file: all partial files first, then rename them in order.
|
|
38
|
+
|
|
39
|
+
Raises FileExistsError if a target exists and `overwrite` is false, OSError if writing
|
|
40
|
+
fails. No partial file is left, and if writing (not renaming) fails no target changed.
|
|
41
|
+
Without `overwrite` the set is all or nothing: if one rename fails (a target appeared
|
|
42
|
+
during the run), the targets already renamed by this call are removed again; they did
|
|
43
|
+
not exist before, so nothing of the person's is deleted. With `overwrite`, a rename
|
|
44
|
+
that fails after earlier ones succeeded cannot be undone (the old files are gone); on
|
|
45
|
+
one file system, after every file was written in full, that is not expected to happen.
|
|
46
|
+
"""
|
|
47
|
+
_write_all(texts, overwrite=overwrite)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _write_all(contents: Mapping[Path, str | bytes], *, overwrite: bool) -> None:
|
|
51
|
+
partials = {target: partial_path(target) for target in contents}
|
|
52
|
+
committed: list[Path] = []
|
|
53
|
+
try:
|
|
54
|
+
for target, content in contents.items():
|
|
55
|
+
_write_partial(partials[target], content)
|
|
56
|
+
for target, partial in partials.items():
|
|
57
|
+
_commit(partial, target, overwrite)
|
|
58
|
+
committed.append(target)
|
|
59
|
+
except BaseException:
|
|
60
|
+
for partial in partials.values():
|
|
61
|
+
with contextlib.suppress(OSError):
|
|
62
|
+
partial.unlink(missing_ok=True)
|
|
63
|
+
if not overwrite:
|
|
64
|
+
for target in committed:
|
|
65
|
+
with contextlib.suppress(OSError):
|
|
66
|
+
target.unlink(missing_ok=True)
|
|
67
|
+
raise
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _write_partial(partial: Path, content: str | bytes) -> None:
|
|
71
|
+
if isinstance(content, bytes):
|
|
72
|
+
with partial.open("wb") as binary:
|
|
73
|
+
binary.write(content)
|
|
74
|
+
binary.flush()
|
|
75
|
+
os.fsync(binary.fileno())
|
|
76
|
+
else:
|
|
77
|
+
with partial.open("w", encoding="utf-8", newline="\n") as text:
|
|
78
|
+
text.write(content)
|
|
79
|
+
text.flush()
|
|
80
|
+
os.fsync(text.fileno())
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _commit(partial: Path, target: Path, overwrite: bool) -> None:
|
|
84
|
+
if overwrite:
|
|
85
|
+
os.replace(partial, target)
|
|
86
|
+
elif sys.platform == "win32":
|
|
87
|
+
os.rename(partial, target) # refuses an existing target on Windows
|
|
88
|
+
else:
|
|
89
|
+
try:
|
|
90
|
+
os.link(partial, target) # refuses an existing target
|
|
91
|
+
except FileExistsError:
|
|
92
|
+
raise
|
|
93
|
+
except OSError: # no hard links here (FAT, some network drives): check, then rename
|
|
94
|
+
if target.exists():
|
|
95
|
+
raise FileExistsError(f"file appeared during the run: {target}") from None
|
|
96
|
+
os.rename(partial, target)
|
|
97
|
+
else:
|
|
98
|
+
partial.unlink()
|