sci2audio 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. sci2audio/__init__.py +3 -0
  2. sci2audio/app/__init__.py +110 -0
  3. sci2audio/app/audio.py +847 -0
  4. sci2audio/app/chapters.py +109 -0
  5. sci2audio/app/chunks.py +102 -0
  6. sci2audio/app/config.py +136 -0
  7. sci2audio/app/estimate.py +192 -0
  8. sci2audio/app/files.py +98 -0
  9. sci2audio/app/job.py +285 -0
  10. sci2audio/app/lock.py +92 -0
  11. sci2audio/app/report.py +400 -0
  12. sci2audio/app/runner.py +92 -0
  13. sci2audio/app/script.py +95 -0
  14. sci2audio/app/settings.py +58 -0
  15. sci2audio/app/voice.py +70 -0
  16. sci2audio/cli.py +1013 -0
  17. sci2audio/core/__init__.py +81 -0
  18. sci2audio/core/audio.py +39 -0
  19. sci2audio/core/content.py +266 -0
  20. sci2audio/core/document.py +225 -0
  21. sci2audio/core/edits.py +121 -0
  22. sci2audio/core/export.py +76 -0
  23. sci2audio/extraction/__init__.py +28 -0
  24. sci2audio/extraction/cover.py +68 -0
  25. sci2audio/extraction/epub.py +576 -0
  26. sci2audio/extraction/errors.py +7 -0
  27. sci2audio/extraction/markdown.py +250 -0
  28. sci2audio/extraction/markup.py +356 -0
  29. sci2audio/extraction/pdf.py +623 -0
  30. sci2audio/extraction/text.py +136 -0
  31. sci2audio/layout/__init__.py +13 -0
  32. sci2audio/layout/build.py +265 -0
  33. sci2audio/layout/captions.py +77 -0
  34. sci2audio/layout/footnotes.py +180 -0
  35. sci2audio/layout/formulas.py +392 -0
  36. sci2audio/layout/furniture.py +326 -0
  37. sci2audio/layout/geometry.py +124 -0
  38. sci2audio/layout/graphics.py +569 -0
  39. sci2audio/layout/headings.py +358 -0
  40. sci2audio/layout/lists.py +167 -0
  41. sci2audio/layout/maths.py +47 -0
  42. sci2audio/layout/order.py +374 -0
  43. sci2audio/layout/paragraphs.py +518 -0
  44. sci2audio/layout/settings.py +142 -0
  45. sci2audio/layout/split.py +113 -0
  46. sci2audio/layout/tagged.py +442 -0
  47. sci2audio/output/__init__.py +44 -0
  48. sci2audio/output/ffmpeg.py +111 -0
  49. sci2audio/output/model.py +125 -0
  50. sci2audio/output/names.py +70 -0
  51. sci2audio/output/readalong.py +160 -0
  52. sci2audio/output/timeline.py +113 -0
  53. sci2audio/output/wav.py +83 -0
  54. sci2audio/output/write.py +280 -0
  55. sci2audio/testing/__init__.py +32 -0
  56. sci2audio/testing/documents.py +124 -0
  57. sci2audio/testing/regression.py +197 -0
  58. sci2audio/testing/text_steps.py +262 -0
  59. sci2audio/testing/voices.py +164 -0
  60. sci2audio/textprep/__init__.py +105 -0
  61. sci2audio/textprep/abbreviations.py +138 -0
  62. sci2audio/textprep/characters.py +267 -0
  63. sci2audio/textprep/citations.py +683 -0
  64. sci2audio/textprep/data/en/abbreviations.toml +209 -0
  65. sci2audio/textprep/data/en/units.toml +548 -0
  66. sci2audio/textprep/footnotes.py +418 -0
  67. sci2audio/textprep/furniture.py +46 -0
  68. sci2audio/textprep/hyphens.py +228 -0
  69. sci2audio/textprep/links.py +628 -0
  70. sci2audio/textprep/notation.py +170 -0
  71. sci2audio/textprep/numbers.py +529 -0
  72. sci2audio/textprep/placeholders.py +68 -0
  73. sci2audio/textprep/references.py +107 -0
  74. sci2audio/textprep/registry.py +83 -0
  75. sci2audio/textprep/runner.py +206 -0
  76. sci2audio/textprep/spoken.py +33 -0
  77. sci2audio/textprep/statistics.py +322 -0
  78. sci2audio/textprep/steps.py +153 -0
  79. sci2audio/textprep/units.py +449 -0
  80. sci2audio/textprep/utterances.py +315 -0
  81. sci2audio/textprep/wording.py +93 -0
  82. sci2audio/textprep/wordlists.py +189 -0
  83. sci2audio/voices/__init__.py +102 -0
  84. sci2audio/voices/download.py +340 -0
  85. sci2audio/voices/engine.py +136 -0
  86. sci2audio/voices/errors.py +88 -0
  87. sci2audio/voices/kokoro_engine.py +346 -0
  88. sci2audio/voices/runtime.py +37 -0
  89. sci2audio/voices/types.py +206 -0
  90. sci2audio/voices/wav.py +82 -0
  91. sci2audio/web/__init__.py +13 -0
  92. sci2audio/web/api.py +658 -0
  93. sci2audio/web/config.py +97 -0
  94. sci2audio/web/documents.py +496 -0
  95. sci2audio/web/i18n.py +188 -0
  96. sci2audio/web/jobs.py +265 -0
  97. sci2audio/web/launch.py +121 -0
  98. sci2audio/web/limits.py +64 -0
  99. sci2audio/web/models.py +178 -0
  100. sci2audio/web/pages.py +509 -0
  101. sci2audio/web/player.py +549 -0
  102. sci2audio/web/security.py +194 -0
  103. sci2audio/web/server.py +116 -0
  104. sci2audio/web/state.py +88 -0
  105. sci2audio/web/static/app.css +150 -0
  106. sci2audio/web/static/app.js +499 -0
  107. sci2audio/web/static/player.js +277 -0
  108. sci2audio/web/strings/de.toml +348 -0
  109. sci2audio/web/strings/en.toml +343 -0
  110. sci2audio/web/tasks.py +280 -0
  111. sci2audio/web/templates/_remove.html +17 -0
  112. sci2audio/web/templates/base.html +32 -0
  113. sci2audio/web/templates/document.html +153 -0
  114. sci2audio/web/templates/index.html +41 -0
  115. sci2audio/web/templates/job.html +66 -0
  116. sci2audio/web/templates/listen.html +75 -0
  117. sci2audio/web/templates/missing.html +7 -0
  118. sci2audio/web/templates/privacy.html +47 -0
  119. sci2audio/web/templates/voice.html +31 -0
  120. sci2audio-1.0.0.dist-info/METADATA +171 -0
  121. sci2audio-1.0.0.dist-info/RECORD +125 -0
  122. sci2audio-1.0.0.dist-info/WHEEL +4 -0
  123. sci2audio-1.0.0.dist-info/entry_points.txt +2 -0
  124. sci2audio-1.0.0.dist-info/licenses/LICENSE +674 -0
  125. sci2audio-1.0.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +86 -0
@@ -0,0 +1,109 @@
1
+ """Chapters: what the person chooses to hear, and what output turns into chapter marks (Q6).
2
+
3
+ - Each top-level section of the document starts a chapter (a level-1 section; a top-level
4
+ section of a deeper level, with no level-1 heading above it, too). A chapter runs to the
5
+ next chapter's start or the document's end, so every block is in exactly one chapter,
6
+ also where the section tree leaves gaps.
7
+ - Text before the first heading is the first chapter, titled with the document title the
8
+ file gives (`Content.metadata.title`), else the file name without its extension. A
9
+ document without headings is one chapter, titled the same way.
10
+ - A chapter whose title, without its printed number, is a reference-list name
11
+ (`REFERENCE_NAMES`) is the reference list: deselected by default (TP-06). Appendices stay
12
+ selected.
13
+
14
+ Titles are kept as printed ("2 Related Work"), white space runs read as one space.
15
+ EPUB's top-level navigation entries become level-1 sections in layout (#73, #75).
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import re
21
+ from collections.abc import Collection
22
+ from dataclasses import dataclass
23
+ from pathlib import PurePath
24
+
25
+ from sci2audio.core import Document
26
+ from sci2audio.textprep import REFERENCE_NAMES
27
+
28
+ # The names a reference-list heading has, compared without case and without a printed
29
+ # number: the reference-list text step's own table, English and German (#221), so the
30
+ # chapter choice and the step agree (D-146). Every language's names count here, since the
31
+ # chapter list does not depend on the job's language.
32
+ _ALL_REFERENCE_NAMES = frozenset(
33
+ name.casefold() for names in REFERENCE_NAMES.values() for name in names
34
+ )
35
+ # a printed heading number before the title: "7", "7.", "A", "A.1", "IV."
36
+ _NUMBER = re.compile(r"\A(?:\d{1,2}|[A-Z]|[IVX]{1,5})(?:\.\d{1,2})*\.?\s+")
37
+ _SPACES = re.compile(r"\s+")
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class Chapter:
42
+ """Chapter number `index` (from 0): its title and blocks `start` up to `end`."""
43
+
44
+ index: int
45
+ title: str
46
+ start: int
47
+ end: int
48
+ references: bool = False # the reference list: deselected by default
49
+
50
+ def __post_init__(self) -> None:
51
+ if not self.title.strip():
52
+ raise ValueError("a chapter needs a title")
53
+ if self.index < 0 or not 0 <= self.start <= self.end:
54
+ raise ValueError(f"chapter {self.title!r} needs index >= 0 and start <= end")
55
+
56
+ def holds(self, block: int) -> bool:
57
+ """True if block number `block` belongs to this chapter."""
58
+ return self.start <= block < self.end
59
+
60
+
61
+ def document_title(document: Document) -> str:
62
+ """The title the file gives, else the file name without its extension ("Untitled" if
63
+ neither has a letter to read)."""
64
+ name = document.content.source.name
65
+ candidates = (document.content.metadata.title, PurePath(name).stem, name)
66
+ return next((title for text in candidates if (title := _clean(text))), "Untitled")
67
+
68
+
69
+ def find_chapters(document: Document) -> tuple[Chapter, ...]:
70
+ """The document's chapters in order; together they hold every block exactly once."""
71
+ count = len(document.blocks)
72
+ starts = [(section.start, section.title) for section in document.sections]
73
+ heads: list[tuple[int, str, bool]] = [] # start, title, reference list
74
+ if not starts or starts[0][0] > 0:
75
+ heads.append((0, document_title(document), False))
76
+ for number, (start, title) in enumerate(starts, start=1):
77
+ cleaned = _clean(title) or f"Section {number}"
78
+ heads.append((start, cleaned, is_reference_title(cleaned)))
79
+ chapters = []
80
+ for index, (start, title, references) in enumerate(heads):
81
+ end = heads[index + 1][0] if index + 1 < len(heads) else count
82
+ chapters.append(Chapter(index, title, start, end, references))
83
+ return tuple(chapters)
84
+
85
+
86
+ def is_reference_title(title: str) -> bool:
87
+ """True if the title names a reference list ("References", "7 Bibliography")."""
88
+ name = _NUMBER.sub("", _clean(title), count=1)
89
+ return name.casefold().rstrip(".:") in _ALL_REFERENCE_NAMES
90
+
91
+
92
+ def default_selection(chapters: Collection[Chapter]) -> frozenset[int]:
93
+ """Every chapter but the reference list."""
94
+ return frozenset(chapter.index for chapter in chapters if not chapter.references)
95
+
96
+
97
+ def selected(
98
+ chapters: Collection[Chapter], chosen: Collection[int] | None
99
+ ) -> tuple[frozenset[int], tuple[int, ...]]:
100
+ """The chosen chapters (the default when `chosen` is None), and the chosen indices the
101
+ document doesn't have, which are ignored."""
102
+ if chosen is None:
103
+ return default_selection(chapters), ()
104
+ known = {chapter.index for chapter in chapters}
105
+ return frozenset(chosen) & known, tuple(sorted(set(chosen) - known))
106
+
107
+
108
+ def _clean(text: str) -> str:
109
+ return _SPACES.sub(" ", text).strip()
@@ -0,0 +1,102 @@
1
+ """The audio job's chunk cache in the working folder (WI-02, D-155, D-156, D-158).
2
+
3
+ Every chunk is the voice's audio of one utterance, cached under a key of what decides the
4
+ sound: the text, the voice, the speed and the engine version (the span is not part of it,
5
+ so the same text twice in a job is spoken once). A chunk is finished when its WAV file is
6
+ complete: the voice writes `<name>.part` and renames it (`voices.write_wav`), so a run
7
+ killed outright leaves at most a `.part` file, which the next run removes. Resuming is
8
+ running the job again on the same working folder: every finished chunk is used again.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import contextlib
14
+ import hashlib
15
+ import json
16
+ import shutil
17
+ import wave
18
+ from pathlib import Path
19
+
20
+ from sci2audio.app.lock import work_folder_lock
21
+
22
+ CHUNKS = "chunks" # the cache's folder inside the working folder
23
+ KEY_FORMAT = 1 # bump when anything else starts to change the sound
24
+ _SAMPLE_WIDTH = 2 # 16-bit PCM (D-117)
25
+
26
+
27
+ def chunk_key(text: str, voice: str, speed: float, engine: str, engine_version: str) -> str:
28
+ """The cache key of a chunk: SHA-256 (hex) of a canonical JSON of what decides its sound."""
29
+ data = {
30
+ "format": KEY_FORMAT,
31
+ "engine": engine,
32
+ "engine_version": engine_version,
33
+ "voice": voice,
34
+ "speed": float(speed),
35
+ "text": text,
36
+ }
37
+ canonical = json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
38
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
39
+
40
+
41
+ def chunk_file(key: str) -> str:
42
+ """The chunk's file relative to the working folder (an `AudioChunk.file`)."""
43
+ return f"{CHUNKS}/{key[:2]}/{key}.wav"
44
+
45
+
46
+ def is_finished(path: Path) -> bool:
47
+ """True if `path` is a complete 16-bit mono PCM WAV file (a finished chunk)."""
48
+ try:
49
+ with wave.open(str(path), "rb") as audio:
50
+ if audio.getnchannels() != 1 or audio.getsampwidth() != _SAMPLE_WIDTH:
51
+ return False
52
+ if audio.getframerate() <= 0:
53
+ return False
54
+ frames = audio.getnframes()
55
+ if frames == 0:
56
+ return True
57
+ audio.setpos(frames - 1) # a file cut short has fewer bytes than its header says
58
+ return len(audio.readframes(1)) == _SAMPLE_WIDTH
59
+ except (OSError, EOFError, wave.Error):
60
+ return False
61
+
62
+
63
+ def remove_partials(work: Path) -> int:
64
+ """Remove the `.part` files a killed run left in the cache; return how many."""
65
+ removed = 0
66
+ folder = work / CHUNKS
67
+ if not folder.is_dir():
68
+ return 0
69
+ for part in folder.glob("*/*.part"):
70
+ with contextlib.suppress(OSError):
71
+ part.unlink()
72
+ removed += 1
73
+ return removed
74
+
75
+
76
+ def cache_size(work: Path) -> int:
77
+ """The bytes the chunk cache takes (0 when there is none)."""
78
+ folder = work / CHUNKS
79
+ if not folder.is_dir():
80
+ return 0
81
+ total = 0
82
+ for path in folder.rglob("*"):
83
+ with contextlib.suppress(OSError):
84
+ if path.is_file():
85
+ total += path.stat().st_size
86
+ return total
87
+
88
+
89
+ def clean_work_folder(work: Path) -> int:
90
+ """Remove the chunk cache from a working folder; return the bytes freed.
91
+
92
+ Only the `chunks` folder goes; the records (`audio.json`, `document.json`, `job.json`)
93
+ and the lock file stay, so a finished job can still be listed and played. The next run
94
+ speaks again. It holds the working folder's lock while it removes the cache, so it never
95
+ removes chunks under a running job: JobBusyError while a job works on the folder (D-226).
96
+ """
97
+ if not work.is_dir():
98
+ return 0 # nothing to clean; the lock would only create the folder
99
+ with work_folder_lock(work):
100
+ size = cache_size(work)
101
+ shutil.rmtree(work / CHUNKS, ignore_errors=True)
102
+ return size
@@ -0,0 +1,136 @@
1
+ """The settings file: the person's choices kept between runs (playbook 3.4).
2
+
3
+ A JSON file in the user's configuration folder (`default_config_file`, from platformdirs).
4
+ Only the command line and the server read and write it; a job gets its settings passed in
5
+ and never touches the file. The chosen chapters are not kept: they belong to one document.
6
+ A missing, unreadable or broken file never stops the program: what can't be read falls
7
+ back to the default, with a warning the caller shows.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import hashlib
13
+ import json
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ import platformdirs
18
+
19
+ from sci2audio.app.files import write_text_atomically
20
+ from sci2audio.app.settings import JobSettings
21
+ from sci2audio.textprep import Category
22
+
23
+ CONFIG_NAME = "settings.json"
24
+ CONFIG_VERSION = 1
25
+ KEPT = ( # every JobSettings field but the chosen chapters
26
+ "language",
27
+ "switched_off",
28
+ "voice",
29
+ "speed",
30
+ "output_folder",
31
+ )
32
+
33
+
34
+ def default_config_file() -> Path:
35
+ """Where the settings file lives: the user's configuration folder for sci2audio.
36
+
37
+ Windows `%APPDATA%\\sci2audio`, macOS `~/Library/Application Support/sci2audio`,
38
+ Linux `$XDG_CONFIG_HOME/sci2audio` (usually `~/.config/sci2audio`).
39
+ """
40
+ folder = platformdirs.user_config_path("sci2audio", appauthor=False, roaming=True)
41
+ return folder / CONFIG_NAME
42
+
43
+
44
+ def default_models_folder() -> Path:
45
+ """Where the voice setup puts the voice files (D-164): the user's data folder, not the
46
+ cache, so a system clean-up never deletes the 355 MB download."""
47
+ return platformdirs.user_data_path("sci2audio", appauthor=False) / "models"
48
+
49
+
50
+ def default_work_folder(source: Path) -> Path:
51
+ """The working folder of an audio job (D-164): one per source file, named by the first
52
+ 16 hex digits of the file's SHA-256, in the user's cache folder. The same file under
53
+ another name or in another folder gets the same folder, so its chunks are used again.
54
+ Raises OSError if the file can't be read."""
55
+ digest = hashlib.sha256()
56
+ with source.open("rb") as file:
57
+ for block in iter(lambda: file.read(1 << 20), b""):
58
+ digest.update(block)
59
+ folder = platformdirs.user_cache_path("sci2audio", appauthor=False)
60
+ return folder / "jobs" / digest.hexdigest()[:16]
61
+
62
+
63
+ def settings_to_data(settings: JobSettings) -> dict[str, Any]:
64
+ """The settings as the file keeps them (without the chosen chapters)."""
65
+ return {
66
+ "version": CONFIG_VERSION,
67
+ "language": settings.language,
68
+ "switched_off": sorted(str(category) for category in settings.switched_off),
69
+ "voice": settings.voice,
70
+ "speed": settings.speed,
71
+ "output_folder": None if settings.output_folder is None else str(settings.output_folder),
72
+ }
73
+
74
+
75
+ def settings_from_data(data: object) -> tuple[JobSettings, list[str]]:
76
+ """Settings from the file's data, and a warning for everything that was left out."""
77
+ if not isinstance(data, dict):
78
+ return JobSettings(), ["the settings file does not hold settings; using the defaults"]
79
+ warnings = []
80
+ version = data.get("version")
81
+ if type(version) is not int or version != CONFIG_VERSION: # true is not version 1
82
+ warnings.append(f"the settings file has version {version!r}; reading what fits")
83
+ values: dict[str, Any] = {}
84
+ for key, value in data.items():
85
+ if key == "version":
86
+ continue
87
+ if key not in KEPT:
88
+ warnings.append(f"unknown setting {key!r} ignored")
89
+ continue
90
+ try:
91
+ converted = _convert(key, value)
92
+ JobSettings(**{key: converted}) # check it on its own
93
+ except (TypeError, ValueError) as error:
94
+ warnings.append(f"setting {key!r} ignored: {error}")
95
+ else:
96
+ values[key] = converted
97
+ return JobSettings(**values), warnings
98
+
99
+
100
+ def load_settings(file: Path) -> tuple[JobSettings, list[str]]:
101
+ """The settings kept in `file`, and warnings; the defaults if there is no file."""
102
+ try:
103
+ text = file.read_text(encoding="utf-8")
104
+ except FileNotFoundError:
105
+ return JobSettings(), []
106
+ except (OSError, UnicodeDecodeError) as error:
107
+ return JobSettings(), [f"the settings file can't be read ({error}); using the defaults"]
108
+ try:
109
+ data = json.loads(text)
110
+ except (ValueError, RecursionError) as error: # JSONDecodeError; nesting too deep
111
+ detail = error if isinstance(error, ValueError) else "nested too deeply"
112
+ return JobSettings(), [f"the settings file is damaged ({detail}); using the defaults"]
113
+ return settings_from_data(data)
114
+
115
+
116
+ def save_settings(settings: JobSettings, file: Path) -> None:
117
+ """Keep the settings in `file`, all or nothing (OSError if it can't be written)."""
118
+ file.parent.mkdir(parents=True, exist_ok=True)
119
+ text = json.dumps(settings_to_data(settings), ensure_ascii=False, indent=2) + "\n"
120
+ write_text_atomically(file, text, overwrite=True)
121
+
122
+
123
+ def _convert(key: str, value: object) -> object:
124
+ if key == "switched_off":
125
+ if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
126
+ raise ValueError(f"expected a list of category names, got {value!r}")
127
+ return frozenset(Category(name) for name in value)
128
+ if key == "output_folder":
129
+ if value is None:
130
+ return None
131
+ if not isinstance(value, str) or not value:
132
+ raise ValueError(f"expected a folder, got {value!r}")
133
+ return Path(value)
134
+ if key in ("language", "voice") and not isinstance(value, str):
135
+ raise ValueError(f"expected text, got {value!r}")
136
+ return value
@@ -0,0 +1,192 @@
1
+ """The audio job's time estimate: what one voice call costs, fitted on this run (#136).
2
+
3
+ A voice call takes a fixed time on top of its characters, and long utterances take more
4
+ per character than short ones, so the seconds of one call are modelled as
5
+
6
+ seconds = constant + per_character * c + per_square * c * c (c = characters)
7
+
8
+ with every coefficient zero or more, fitted by least squares on the chunks measured so far
9
+ in this run (D-380). The time left is that cost summed over every chunk still to speak.
10
+
11
+ What keeps a curve from running wild:
12
+ - fewer than MIN_CHUNKS measured chunks (twice the coefficients), or fewer different
13
+ lengths than coefficients: the plain rate (measured seconds / measured characters,
14
+ D-159) instead of the fit;
15
+ - the curve is never used beyond the longest measured chunk L: a longer chunk costs
16
+ model(L) / L per character, the cost per character at the edge of what was measured
17
+ (D-384). A first chapter of names, headings or a list measures only short chunks,
18
+ and the curve fitted on them can say almost twice the real time for long paragraphs.
19
+ The fit also keeps the plain rate when it explains the measured chunks no better than the
20
+ rate does; since the per-character-only fit is one of its candidates, that happens only
21
+ when the cost is exactly per character.
22
+
23
+ The measured chunks are kept as running sums, so adding one is constant time; the chunks
24
+ still to speak are counted by length, so the time left costs one step per different
25
+ length (at most the voice's longest utterance, 300 characters by default).
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import itertools
31
+ from collections import Counter
32
+ from dataclasses import dataclass, field
33
+
34
+ TERMS = 3 # constant, per character, per character squared
35
+ MIN_CHUNKS = 2 * TERMS # measured chunks before the fit is trusted over the plain rate
36
+ SCALE = 100.0 # characters are fitted in hundreds, so the sums stay well conditioned
37
+ _PIVOT = 1e-9 # a smaller pivot (relative to the diagonal) means the terms cannot be told apart
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class CostModel:
42
+ """Seconds one voice call takes for `characters` characters."""
43
+
44
+ constant: float = 0.0
45
+ per_character: float = 0.0
46
+ per_square: float = 0.0
47
+
48
+ def seconds(self, characters: int) -> float:
49
+ c = float(characters)
50
+ return self.constant + self.per_character * c + self.per_square * c * c
51
+
52
+ @classmethod
53
+ def rate(cls, seconds: float, characters: int) -> CostModel:
54
+ """The plain rate: every character costs the same, nothing per call."""
55
+ return cls(per_character=seconds / characters)
56
+
57
+
58
+ @dataclass
59
+ class _Sums:
60
+ """Running sums of x**k (k = 0..2*TERMS-2) and y*x**k (k = 0..TERMS-1) and y*y."""
61
+
62
+ powers: list[float] = field(default_factory=lambda: [0.0] * (2 * TERMS - 1))
63
+ weighted: list[float] = field(default_factory=lambda: [0.0] * TERMS)
64
+ squares: float = 0.0
65
+
66
+ def add(self, x: float, y: float) -> None:
67
+ for k in range(len(self.powers)):
68
+ self.powers[k] += x**k
69
+ for k in range(TERMS):
70
+ self.weighted[k] += y * x**k
71
+ self.squares += y * y
72
+
73
+
74
+ @dataclass
75
+ class Estimate:
76
+ """The measured chunks of one run and the chunks still to speak."""
77
+
78
+ measured_characters: int = 0
79
+ measured_seconds: float = 0.0
80
+ measured_chunks: int = 0
81
+ _measured: _Sums = field(default_factory=_Sums)
82
+ _lengths: set[int] = field(default_factory=set)
83
+ _left: Counter[int] = field(default_factory=Counter) # length -> chunks still to speak
84
+
85
+ def measure(self, characters: int, seconds: float) -> None:
86
+ """One voice call of `characters` characters took `seconds`."""
87
+ self.measured_characters += characters
88
+ self.measured_seconds += seconds
89
+ self.measured_chunks += 1
90
+ self._measured.add(characters / SCALE, seconds)
91
+ self._lengths.add(characters)
92
+
93
+ def expect(self, characters: int) -> None:
94
+ """A chunk of `characters` characters is still to be spoken."""
95
+ self._left[characters] += 1
96
+
97
+ def done(self, characters: int) -> None:
98
+ """A chunk `expect`ed earlier is spoken (or failed): it is no longer left."""
99
+ self._left[characters] -= 1
100
+ if self._left[characters] <= 0:
101
+ del self._left[characters]
102
+
103
+ def speed(self) -> float | None:
104
+ """Measured characters per second, the plain rate; None before any measurement."""
105
+ if self.measured_characters == 0 or self.measured_seconds <= 0:
106
+ return None
107
+ return self.measured_characters / self.measured_seconds
108
+
109
+ def model(self) -> CostModel | None:
110
+ """The cost of one call, fitted on the measured chunks; None before any."""
111
+ if self.speed() is None:
112
+ return None
113
+ rate = CostModel.rate(self.measured_seconds, self.measured_characters)
114
+ if self.measured_chunks < MIN_CHUNKS or len(self._lengths) < TERMS:
115
+ return rate
116
+ fitted = _fit(self._measured)
117
+ if fitted is None:
118
+ return rate
119
+ coefficients, error = fitted
120
+ if error >= self._error(rate):
121
+ return rate
122
+ constant, per_character, per_square = coefficients
123
+ return CostModel(constant, per_character / SCALE, per_square / SCALE**2)
124
+
125
+ def seconds_left(self) -> float | None:
126
+ """The model summed over every chunk still to speak, never beyond the longest
127
+ measured chunk; None before any measurement."""
128
+ model = self.model()
129
+ if model is None:
130
+ return None
131
+ longest = max(self._lengths)
132
+ per_character_beyond = model.seconds(longest) / longest
133
+ seconds = sum(
134
+ count * (model.seconds(length) if length <= longest else per_character_beyond * length)
135
+ for length, count in self._left.items()
136
+ )
137
+ return max(0.0, seconds)
138
+
139
+ def _error(self, model: CostModel) -> float:
140
+ """Sum of squared errors of `model` over the measured chunks."""
141
+ p = (model.constant, model.per_character * SCALE, model.per_square * SCALE**2)
142
+ return _sse(self._measured, p)
143
+
144
+
145
+ def _sse(sums: _Sums, p: tuple[float, ...]) -> float:
146
+ """Sum over the measured chunks of (y - sum p_k x**k)**2, from the running sums."""
147
+ cross = sum(p[k] * sums.weighted[k] for k in range(TERMS))
148
+ quad = sum(p[i] * p[j] * sums.powers[i + j] for i in range(TERMS) for j in range(TERMS))
149
+ return max(0.0, sums.squares - 2 * cross + quad)
150
+
151
+
152
+ def _fit(sums: _Sums) -> tuple[tuple[float, ...], float] | None:
153
+ """Non-negative least squares over the terms, exactly: the best unconstrained fit of
154
+ every subset of terms whose coefficients all come out zero or more. None if no subset
155
+ can be solved."""
156
+ best: tuple[tuple[float, ...], float] | None = None
157
+ for size in range(1, TERMS + 1):
158
+ for subset in itertools.combinations(range(TERMS), size):
159
+ solved = _solve(
160
+ [[sums.powers[i + j] for j in subset] for i in subset],
161
+ [sums.weighted[i] for i in subset],
162
+ )
163
+ if solved is None or any(value < 0 for value in solved):
164
+ continue
165
+ p = [0.0] * TERMS
166
+ for index, value in zip(subset, solved, strict=True):
167
+ p[index] = value
168
+ error = _sse(sums, tuple(p))
169
+ if best is None or error < best[1]:
170
+ best = (tuple(p), error)
171
+ return best
172
+
173
+
174
+ def _solve(matrix: list[list[float]], vector: list[float]) -> list[float] | None:
175
+ """Gaussian elimination with partial pivoting; None if the matrix is (nearly) singular."""
176
+ n = len(vector)
177
+ rows = [[*matrix[i], vector[i]] for i in range(n)]
178
+ scale = max(abs(rows[i][i]) for i in range(n)) or 1.0
179
+ for column in range(n):
180
+ pivot = max(range(column, n), key=lambda r: abs(rows[r][column]))
181
+ if abs(rows[pivot][column]) <= _PIVOT * scale:
182
+ return None
183
+ rows[column], rows[pivot] = rows[pivot], rows[column]
184
+ for r in range(column + 1, n):
185
+ factor = rows[r][column] / rows[column][column]
186
+ for k in range(column, n + 1):
187
+ rows[r][k] -= factor * rows[column][k]
188
+ solution = [0.0] * n
189
+ for r in reversed(range(n)):
190
+ total = rows[r][n] - sum(rows[r][k] * solution[k] for k in range(r + 1, n))
191
+ solution[r] = total / rows[r][r]
192
+ return solution
sci2audio/app/files.py ADDED
@@ -0,0 +1,98 @@
1
+ """Writing the job's files so that no half-written file is ever left (the #69 rule).
2
+
3
+ Each file is written under a temporary name (`.<name>.partial`) next to its target and
4
+ renamed when complete; on any error the partial files are removed. `write_texts` writes
5
+ all its files before it renames any, so a full disk leaves every earlier file as it was,
6
+ and without `overwrite` it puts either all of its files in place or none.
7
+ An existing file is only replaced with `overwrite=True`; without it, a file that appeared
8
+ at the target during the run is never replaced either. A run killed outright may leave a
9
+ partial file behind; the next run writes over it.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import contextlib
15
+ import os
16
+ import sys
17
+ from collections.abc import Mapping
18
+ from pathlib import Path
19
+
20
+
21
+ def partial_path(target: Path) -> Path:
22
+ """The temporary name a file is written under."""
23
+ return target.with_name(f".{target.name}.partial")
24
+
25
+
26
+ def write_text_atomically(target: Path, text: str, *, overwrite: bool) -> None:
27
+ """Write `text` (UTF-8, "\\n" line ends) to `target`, all of it or nothing."""
28
+ write_texts({target: text}, overwrite=overwrite)
29
+
30
+
31
+ def write_bytes_atomically(target: Path, data: bytes, *, overwrite: bool) -> None:
32
+ """Write `data` to `target`, all of it or nothing (as `write_texts`)."""
33
+ _write_all({target: data}, overwrite=overwrite)
34
+
35
+
36
+ def write_texts(texts: Mapping[Path, str], *, overwrite: bool) -> None:
37
+ """Write each text to its file: all partial files first, then rename them in order.
38
+
39
+ Raises FileExistsError if a target exists and `overwrite` is false, OSError if writing
40
+ fails. No partial file is left, and if writing (not renaming) fails no target changed.
41
+ Without `overwrite` the set is all or nothing: if one rename fails (a target appeared
42
+ during the run), the targets already renamed by this call are removed again; they did
43
+ not exist before, so nothing of the person's is deleted. With `overwrite`, a rename
44
+ that fails after earlier ones succeeded cannot be undone (the old files are gone); on
45
+ one file system, after every file was written in full, that is not expected to happen.
46
+ """
47
+ _write_all(texts, overwrite=overwrite)
48
+
49
+
50
+ def _write_all(contents: Mapping[Path, str | bytes], *, overwrite: bool) -> None:
51
+ partials = {target: partial_path(target) for target in contents}
52
+ committed: list[Path] = []
53
+ try:
54
+ for target, content in contents.items():
55
+ _write_partial(partials[target], content)
56
+ for target, partial in partials.items():
57
+ _commit(partial, target, overwrite)
58
+ committed.append(target)
59
+ except BaseException:
60
+ for partial in partials.values():
61
+ with contextlib.suppress(OSError):
62
+ partial.unlink(missing_ok=True)
63
+ if not overwrite:
64
+ for target in committed:
65
+ with contextlib.suppress(OSError):
66
+ target.unlink(missing_ok=True)
67
+ raise
68
+
69
+
70
+ def _write_partial(partial: Path, content: str | bytes) -> None:
71
+ if isinstance(content, bytes):
72
+ with partial.open("wb") as binary:
73
+ binary.write(content)
74
+ binary.flush()
75
+ os.fsync(binary.fileno())
76
+ else:
77
+ with partial.open("w", encoding="utf-8", newline="\n") as text:
78
+ text.write(content)
79
+ text.flush()
80
+ os.fsync(text.fileno())
81
+
82
+
83
+ def _commit(partial: Path, target: Path, overwrite: bool) -> None:
84
+ if overwrite:
85
+ os.replace(partial, target)
86
+ elif sys.platform == "win32":
87
+ os.rename(partial, target) # refuses an existing target on Windows
88
+ else:
89
+ try:
90
+ os.link(partial, target) # refuses an existing target
91
+ except FileExistsError:
92
+ raise
93
+ except OSError: # no hard links here (FAT, some network drives): check, then rename
94
+ if target.exists():
95
+ raise FileExistsError(f"file appeared during the run: {target}") from None
96
+ os.rename(partial, target)
97
+ else:
98
+ partial.unlink()