tiny-voca-ru 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tiny_voca_ru/__init__.py +78 -0
- tiny_voca_ru/checkpoint.py +135 -0
- tiny_voca_ru/config.py +101 -0
- tiny_voca_ru/export.py +374 -0
- tiny_voca_ru/frontend/__init__.py +96 -0
- tiny_voca_ru/frontend/clean.py +90 -0
- tiny_voca_ru/frontend/data/LICENSE-ruaccent +15 -0
- tiny_voca_ru/frontend/data/accent.onnx +0 -0
- tiny_voca_ru/frontend/data/stress.txt.gz +0 -0
- tiny_voca_ru/frontend/g2p.py +381 -0
- tiny_voca_ru/frontend/lexicon.py +218 -0
- tiny_voca_ru/frontend/normalize.py +856 -0
- tiny_voca_ru/frontend/numbers.py +387 -0
- tiny_voca_ru/frontend/stress.py +428 -0
- tiny_voca_ru/frontend/symbols.py +103 -0
- tiny_voca_ru/frontend/tables.py +784 -0
- tiny_voca_ru/frontend/tokenize.py +222 -0
- tiny_voca_ru/infer.py +868 -0
- tiny_voca_ru/model.py +525 -0
- tiny_voca_ru/train/__init__.py +58 -0
- tiny_voca_ru/train/align.py +374 -0
- tiny_voca_ru/train/dataset.py +217 -0
- tiny_voca_ru/train/losses.py +214 -0
- tiny_voca_ru/train/prepare.py +373 -0
- tiny_voca_ru/train/train.py +655 -0
- tiny_voca_ru-0.1.0.dist-info/METADATA +252 -0
- tiny_voca_ru-0.1.0.dist-info/RECORD +33 -0
- tiny_voca_ru-0.1.0.dist-info/WHEEL +5 -0
- tiny_voca_ru-0.1.0.dist-info/entry_points.txt +6 -0
- tiny_voca_ru-0.1.0.dist-info/licenses/LICENSE +202 -0
- tiny_voca_ru-0.1.0.dist-info/licenses/NOTICE +24 -0
- tiny_voca_ru-0.1.0.dist-info/licenses/src/tiny_voca_ru/frontend/data/LICENSE-ruaccent +15 -0
- tiny_voca_ru-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Pure-Python Russian text frontend: normalization, lexical stress, G2P, and tokenization."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .clean import NotNormalized, clean
|
|
6
|
+
from .g2p import phrase_phones
|
|
7
|
+
from .normalize import normalize
|
|
8
|
+
from .stress import (
|
|
9
|
+
LiteStress,
|
|
10
|
+
NoStress,
|
|
11
|
+
RuaccentStress,
|
|
12
|
+
StressProvider,
|
|
13
|
+
accent_input_ids,
|
|
14
|
+
stress_from_logits,
|
|
15
|
+
)
|
|
16
|
+
from .symbols import (
|
|
17
|
+
BOS,
|
|
18
|
+
CONSONANTS,
|
|
19
|
+
EOS,
|
|
20
|
+
PAD,
|
|
21
|
+
PAUSABLE,
|
|
22
|
+
PHONE_IDS,
|
|
23
|
+
PHONE_SET,
|
|
24
|
+
PHONES,
|
|
25
|
+
PUNCTUATION,
|
|
26
|
+
SP,
|
|
27
|
+
SPECIALS,
|
|
28
|
+
SYMBOL_TO_ID,
|
|
29
|
+
SYMBOLS,
|
|
30
|
+
UNK,
|
|
31
|
+
VOICELESS,
|
|
32
|
+
VOWEL_IDS,
|
|
33
|
+
VOWEL_SET,
|
|
34
|
+
VOWELS,
|
|
35
|
+
)
|
|
36
|
+
from .tokenize import (
|
|
37
|
+
NothingToPronounce,
|
|
38
|
+
PhoneSequence,
|
|
39
|
+
accentuate,
|
|
40
|
+
encode_text,
|
|
41
|
+
ensure_terminal_punctuation,
|
|
42
|
+
is_phone,
|
|
43
|
+
phonemize,
|
|
44
|
+
pieces,
|
|
45
|
+
split_clause,
|
|
46
|
+
split_sentences,
|
|
47
|
+
stress_provider,
|
|
48
|
+
tokenize,
|
|
49
|
+
transcribe,
|
|
50
|
+
unknown_words,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"BOS",
|
|
55
|
+
"CONSONANTS",
|
|
56
|
+
"EOS",
|
|
57
|
+
"PAD",
|
|
58
|
+
"PAUSABLE",
|
|
59
|
+
"PHONES",
|
|
60
|
+
"PHONE_IDS",
|
|
61
|
+
"PHONE_SET",
|
|
62
|
+
"PUNCTUATION",
|
|
63
|
+
"SP",
|
|
64
|
+
"SPECIALS",
|
|
65
|
+
"SYMBOLS",
|
|
66
|
+
"SYMBOL_TO_ID",
|
|
67
|
+
"UNK",
|
|
68
|
+
"VOICELESS",
|
|
69
|
+
"VOWELS",
|
|
70
|
+
"VOWEL_IDS",
|
|
71
|
+
"VOWEL_SET",
|
|
72
|
+
"LiteStress",
|
|
73
|
+
"NoStress",
|
|
74
|
+
"NotNormalized",
|
|
75
|
+
"NothingToPronounce",
|
|
76
|
+
"PhoneSequence",
|
|
77
|
+
"RuaccentStress",
|
|
78
|
+
"StressProvider",
|
|
79
|
+
"accent_input_ids",
|
|
80
|
+
"accentuate",
|
|
81
|
+
"clean",
|
|
82
|
+
"encode_text",
|
|
83
|
+
"ensure_terminal_punctuation",
|
|
84
|
+
"is_phone",
|
|
85
|
+
"normalize",
|
|
86
|
+
"phonemize",
|
|
87
|
+
"phrase_phones",
|
|
88
|
+
"pieces",
|
|
89
|
+
"split_clause",
|
|
90
|
+
"split_sentences",
|
|
91
|
+
"stress_from_logits",
|
|
92
|
+
"stress_provider",
|
|
93
|
+
"tokenize",
|
|
94
|
+
"transcribe",
|
|
95
|
+
"unknown_words",
|
|
96
|
+
]
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Unicode cleanup and stress-mark canonicalization (U+0301 -> ``+``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import unicodedata
|
|
7
|
+
|
|
8
|
+
from .symbols import PUNCTUATION
|
|
9
|
+
|
|
10
|
+
LETTER = "а-яёА-ЯЁ"
|
|
11
|
+
VOWELS = "аеёиоуыэюяАЕЁИОУЫЭЮЯ"
|
|
12
|
+
QUOTES = str.maketrans(
|
|
13
|
+
{
|
|
14
|
+
"«": '"',
|
|
15
|
+
"»": '"',
|
|
16
|
+
"„": '"',
|
|
17
|
+
"“": '"',
|
|
18
|
+
"”": '"',
|
|
19
|
+
"‟": '"',
|
|
20
|
+
"‘": "'",
|
|
21
|
+
"’": "'",
|
|
22
|
+
"‚": "'",
|
|
23
|
+
"‛": "'",
|
|
24
|
+
"`": "'",
|
|
25
|
+
"´": "'",
|
|
26
|
+
"[": "(",
|
|
27
|
+
"]": ")",
|
|
28
|
+
"{": "(",
|
|
29
|
+
"}": ")",
|
|
30
|
+
"−": "-",
|
|
31
|
+
"‐": "-",
|
|
32
|
+
"‑": "-",
|
|
33
|
+
"‒": "–",
|
|
34
|
+
"―": "—",
|
|
35
|
+
"\u00a0": " ",
|
|
36
|
+
"\u2009": " ",
|
|
37
|
+
"\u202f": " ",
|
|
38
|
+
"\u2007": " ",
|
|
39
|
+
"℃": "°C",
|
|
40
|
+
"¼": "1/4",
|
|
41
|
+
"½": "1/2",
|
|
42
|
+
"¾": "3/4",
|
|
43
|
+
"⅓": "1/3",
|
|
44
|
+
"⅔": "2/3",
|
|
45
|
+
}
|
|
46
|
+
)
|
|
47
|
+
_PUNCTUATION = "".join(re.escape(p) for p in PUNCTUATION)
|
|
48
|
+
SPOKEN_SIGNS = "%$€₽£¥₴@&=№§°×~≈^"
|
|
49
|
+
_UNSPOKEN = re.compile(rf"\S*(?:[0-9A-Za-z{re.escape(SPOKEN_SIGNS)}]|\+(?![{VOWELS}]))\S*")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class NotNormalized(ValueError):
|
|
53
|
+
"""Raised when text contains digits, Latin letters, or signs needing normalization."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def unify(text: str) -> str:
|
|
57
|
+
"""Normalize Unicode, quotes, dashes, spaces, ellipsis, and combining acute accents."""
|
|
58
|
+
text = unicodedata.normalize("NFC", text).translate(QUOTES)
|
|
59
|
+
text = re.sub(rf"([{VOWELS}])[\u0301\u0300]", r"+\1", text) # за́мок -> з+амок
|
|
60
|
+
text = re.sub(
|
|
61
|
+
r"[\u00c0-\u024f]", lambda m: unicodedata.normalize("NFD", m[0])[0], text
|
|
62
|
+
) # café -> cafe
|
|
63
|
+
text = re.sub(r"[\u0300-\u036f]", "", text)
|
|
64
|
+
return re.sub(r"\.{3,}", "…", text)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def unspoken(text: str) -> list[str]:
|
|
68
|
+
"""Return whitespace-delimited tokens that require text normalization."""
|
|
69
|
+
return _UNSPOKEN.findall(text)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def tidy(text: str) -> str:
|
|
73
|
+
"""Normalize dashes, hyphens, stray symbols, and whitespace."""
|
|
74
|
+
text = re.sub(r"-{2,}|–", "—", text)
|
|
75
|
+
text = re.sub(rf"(?<![{LETTER}+])-|-(?![{LETTER}+])", " — ", text) # non-word hyphen -> dash
|
|
76
|
+
text = re.sub(rf"[^\s{LETTER}+\-{_PUNCTUATION}]", " ", text)
|
|
77
|
+
text = re.sub(r"(?<=\S)—", " —", text)
|
|
78
|
+
text = re.sub(r"—(?=\S)", "— ", text)
|
|
79
|
+
text = re.sub(r"(?<=[?!])\.\.(?!\.)", "", text)
|
|
80
|
+
text = re.sub(r"(?<!\.)\.\.(?!\.)", ".", text)
|
|
81
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def clean(text: str) -> str:
|
|
85
|
+
"""Clean already-normalized text; raise ``NotNormalized`` on digits/Latin/signs."""
|
|
86
|
+
text = unify(text)
|
|
87
|
+
found = unspoken(text)
|
|
88
|
+
if found:
|
|
89
|
+
raise NotNormalized(f"needs a normaliser: {' '.join(found)}")
|
|
90
|
+
return tidy(text)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
stress.txt.gz is a subset of the stress dictionary of RUAccent and accent.onnx
|
|
2
|
+
is its nn_accent/model.onnx, both from https://huggingface.co/ruaccent/accentuator
|
|
3
|
+
(revision b78ae5ea1e62beaf138bed1865cd8c3b0b5ca855), https://github.com/Den4ikAI/ruaccent.
|
|
4
|
+
They are distributed under the MIT License below, not under this project's licence.
|
|
5
|
+
stress.txt.gz is rebuilt by scripts/build_stress_lexicon.py.
|
|
6
|
+
|
|
7
|
+
MIT License
|
|
8
|
+
|
|
9
|
+
Copyright 2026 Denis Petrov
|
|
10
|
+
|
|
11
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
"""Russian stressed orthography to IPA phoneme sequences."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterator, Sequence
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
from .lexicon import (
|
|
10
|
+
ADJECTIVE_ENDINGS,
|
|
11
|
+
ADJECTIVE_STEMS,
|
|
12
|
+
ENCLITICS,
|
|
13
|
+
KEEP_G,
|
|
14
|
+
NONREDUCING,
|
|
15
|
+
NOUN_ENDINGS,
|
|
16
|
+
NOUN_STEMS,
|
|
17
|
+
PROCLITICS,
|
|
18
|
+
WORDS,
|
|
19
|
+
)
|
|
20
|
+
from .symbols import OBSTRUENTS, SOFT, VOICED_PARTNER, VOICELESS, VOICELESS_PARTNER, VOWEL_SET
|
|
21
|
+
|
|
22
|
+
STRESS = "+"
|
|
23
|
+
VOWEL_LETTERS: frozenset[str] = frozenset("аеёиоуыэюя")
|
|
24
|
+
VOWEL_PHONES: dict[str, str] = {
|
|
25
|
+
"а": "a",
|
|
26
|
+
"я": "a",
|
|
27
|
+
"о": "o",
|
|
28
|
+
"ё": "o",
|
|
29
|
+
"у": "u",
|
|
30
|
+
"ю": "u",
|
|
31
|
+
"э": "e",
|
|
32
|
+
"е": "e",
|
|
33
|
+
"ы": "ɨ",
|
|
34
|
+
"и": "i",
|
|
35
|
+
}
|
|
36
|
+
CONSONANT_PHONES: dict[str, str] = {
|
|
37
|
+
"б": "b",
|
|
38
|
+
"в": "v",
|
|
39
|
+
"г": "ɡ",
|
|
40
|
+
"д": "d",
|
|
41
|
+
"ж": "ʐ",
|
|
42
|
+
"з": "z",
|
|
43
|
+
"й": "j",
|
|
44
|
+
"к": "k",
|
|
45
|
+
"л": "l",
|
|
46
|
+
"м": "m",
|
|
47
|
+
"н": "n",
|
|
48
|
+
"п": "p",
|
|
49
|
+
"р": "r",
|
|
50
|
+
"с": "s",
|
|
51
|
+
"т": "t",
|
|
52
|
+
"ф": "f",
|
|
53
|
+
"х": "x",
|
|
54
|
+
"ц": "ts",
|
|
55
|
+
"ч": "tɕ",
|
|
56
|
+
"ш": "ʂ",
|
|
57
|
+
"щ": "ɕ",
|
|
58
|
+
}
|
|
59
|
+
SOFTENING: frozenset[str] = frozenset("еёиюяь")
|
|
60
|
+
IOTATED: frozenset[str] = frozenset("яюеё")
|
|
61
|
+
HUSHING: frozenset[str] = frozenset("жшц") # always hard: и -> ɨ, unstressed е -> ɨ
|
|
62
|
+
SOFT_PHONES: frozenset[str] = frozenset(SOFT.values()) | {"tɕ", "ɕ", "j"}
|
|
63
|
+
|
|
64
|
+
# Ordered consonant-cluster respellings preserving vowel letter count.
|
|
65
|
+
CLUSTERS: tuple[tuple[re.Pattern[str], str], ...] = tuple(
|
|
66
|
+
(re.compile(pattern), replacement)
|
|
67
|
+
for pattern, replacement in (
|
|
68
|
+
(r"ть?ся$", "ца"), # учится, учиться
|
|
69
|
+
(r"вств", "ств"), # чувство, здравствуй
|
|
70
|
+
(r"стск", "ск"), # туристский
|
|
71
|
+
(r"н[тд](?=ск|ц)", "н"), # гигантский, голландцы
|
|
72
|
+
(r"нтг", "нг"), # рентген
|
|
73
|
+
(r"ндш", "нш"), # ландшафт
|
|
74
|
+
(r"стн", "сн"), # честный
|
|
75
|
+
(r"здн", "зн"), # поздно
|
|
76
|
+
(r"стл", "сл"), # счастливый
|
|
77
|
+
(r"лнц", "нц"), # солнце
|
|
78
|
+
(r"рдц", "рц"), # сердце
|
|
79
|
+
(r"[тд]ств", "цтв"), # детство
|
|
80
|
+
(r"[тд]ск(?=(ий|ая|ое|ие|ого|ому|ом|ой|ую|их|им|ими|а|у|е)?$)", "цк"), # детский, городской
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
_NOT_LETTER = re.compile(r"[^а-яё+]")
|
|
84
|
+
_STRAY_STRESS = re.compile(r"\+(?![аеёиоуыэюя])")
|
|
85
|
+
_HARD_E = re.compile(r"[бвгджзклмнпрстфхц]\+?э")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass
|
|
89
|
+
class Unit:
|
|
90
|
+
"""One written word or hyphenated component."""
|
|
91
|
+
|
|
92
|
+
text: str # with stress marks
|
|
93
|
+
key: str # without stress marks
|
|
94
|
+
hyphen: bool # follows a hyphen: кто-то, скажи-ка
|
|
95
|
+
stressed: bool
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass
|
|
99
|
+
class Segment:
|
|
100
|
+
"""Intermediate phonetic segment during phonological rule application."""
|
|
101
|
+
|
|
102
|
+
phone: str
|
|
103
|
+
letter: str = "" # source letter for consonants (в is transparent to voicing)
|
|
104
|
+
context: str = "" # vowels: "soft", "hard", "hush" (after ж ш ц), "" (no preceding C)
|
|
105
|
+
stressed: bool = False
|
|
106
|
+
full: bool = False # retains quality when unstressed
|
|
107
|
+
last: bool = False # final vowel of its written word
|
|
108
|
+
initial: bool = False # first segment of its written word
|
|
109
|
+
loan: bool = False # word contains hard е (respelled э after consonant)
|
|
110
|
+
unit: int = 0
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def respell(word: str) -> str:
|
|
114
|
+
"""Apply lexicon and cluster respellings, preserving stress by vowel index."""
|
|
115
|
+
plain = word.replace(STRESS, "")
|
|
116
|
+
spelled = WORDS.get(plain) or _stem(plain) or plain
|
|
117
|
+
for pattern, replacement in CLUSTERS:
|
|
118
|
+
spelled = pattern.sub(replacement, spelled)
|
|
119
|
+
if spelled.endswith(("ого", "его")) and plain not in KEEP_G:
|
|
120
|
+
spelled = spelled[:-2] + "во" # нового, его: genitive г -> в
|
|
121
|
+
return _restress(word, spelled) if spelled != plain else word
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _stem(plain: str) -> str | None:
|
|
125
|
+
for cut in range(len(plain), 0, -1):
|
|
126
|
+
stem, ending = plain[:cut], plain[cut:]
|
|
127
|
+
if stem in NOUN_STEMS and ending in NOUN_ENDINGS:
|
|
128
|
+
return NOUN_STEMS[stem] + ending
|
|
129
|
+
if stem in ADJECTIVE_STEMS and ending in ADJECTIVE_ENDINGS:
|
|
130
|
+
return ADJECTIVE_STEMS[stem] + ending
|
|
131
|
+
return None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _restress(source: str, target: str) -> str:
|
|
135
|
+
marks: set[int] = set()
|
|
136
|
+
count = 0
|
|
137
|
+
mark = False
|
|
138
|
+
for char in source:
|
|
139
|
+
if char == STRESS:
|
|
140
|
+
mark = True
|
|
141
|
+
elif char in VOWEL_LETTERS:
|
|
142
|
+
if mark:
|
|
143
|
+
marks.add(count)
|
|
144
|
+
count, mark = count + 1, False
|
|
145
|
+
out: list[str] = []
|
|
146
|
+
count = 0
|
|
147
|
+
for char in target:
|
|
148
|
+
if char in VOWEL_LETTERS:
|
|
149
|
+
if count in marks:
|
|
150
|
+
out.append(STRESS)
|
|
151
|
+
count += 1
|
|
152
|
+
out.append(char)
|
|
153
|
+
return "".join(out)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def spell(word: str) -> list[Segment]:
|
|
157
|
+
"""Convert letters (with ``+`` stress marks) of one word into raw segments."""
|
|
158
|
+
letters: list[str] = []
|
|
159
|
+
stressed: set[int] = set()
|
|
160
|
+
for char in word:
|
|
161
|
+
if char == STRESS:
|
|
162
|
+
stressed.add(len(letters))
|
|
163
|
+
else:
|
|
164
|
+
letters.append(char)
|
|
165
|
+
if not stressed:
|
|
166
|
+
stressed = {i for i, char in enumerate(letters) if char == "ё"}
|
|
167
|
+
out: list[Segment] = []
|
|
168
|
+
for i, char in enumerate(letters):
|
|
169
|
+
before = letters[i - 1] if i else ""
|
|
170
|
+
after = letters[i + 1] if i + 1 < len(letters) else ""
|
|
171
|
+
if char in CONSONANT_PHONES:
|
|
172
|
+
phone = CONSONANT_PHONES[char]
|
|
173
|
+
if phone in SOFT and after in SOFTENING:
|
|
174
|
+
phone = SOFT[phone]
|
|
175
|
+
out.append(Segment(phone, char))
|
|
176
|
+
elif char in VOWEL_LETTERS:
|
|
177
|
+
quality = VOWEL_PHONES[char]
|
|
178
|
+
if before in CONSONANT_PHONES:
|
|
179
|
+
if before in HUSHING:
|
|
180
|
+
context = "hard" if char == "э" else "hush" # э only in respelled loanwords
|
|
181
|
+
if char == "и":
|
|
182
|
+
quality = "ɨ"
|
|
183
|
+
else:
|
|
184
|
+
context = "soft" if out[-1].phone in SOFT_PHONES else "hard"
|
|
185
|
+
elif char in IOTATED or (before in ("ь", "ъ") and char in ("и", "о")):
|
|
186
|
+
out.append(Segment("j"))
|
|
187
|
+
context = "soft"
|
|
188
|
+
else:
|
|
189
|
+
context = ""
|
|
190
|
+
out.append(
|
|
191
|
+
Segment(quality, char, context, i in stressed, char == "ё" and i not in stressed)
|
|
192
|
+
)
|
|
193
|
+
vowels = [s for s in out if s.phone in VOWEL_SET]
|
|
194
|
+
if vowels:
|
|
195
|
+
vowels[-1].last = True
|
|
196
|
+
if out:
|
|
197
|
+
out[0].initial = True
|
|
198
|
+
return out
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _units(word: str) -> Iterator[Unit]:
|
|
202
|
+
for index, part in enumerate(word.lower().split("-")):
|
|
203
|
+
text = _STRAY_STRESS.sub("", _NOT_LETTER.sub("", part))
|
|
204
|
+
key = text.replace(STRESS, "")
|
|
205
|
+
if key:
|
|
206
|
+
yield Unit(text, key, index > 0, STRESS in text or "ё" in key)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _groups(units: Sequence[Unit]) -> list[list[Unit]]:
|
|
210
|
+
"""Group written units into phonetic words (each stressed host with its clitics)."""
|
|
211
|
+
groups: list[list[Unit]] = []
|
|
212
|
+
pending: list[Unit] = []
|
|
213
|
+
for unit in units:
|
|
214
|
+
if unit.stressed:
|
|
215
|
+
groups.append([*pending, unit])
|
|
216
|
+
pending = []
|
|
217
|
+
elif (
|
|
218
|
+
groups
|
|
219
|
+
and not pending
|
|
220
|
+
and (
|
|
221
|
+
unit.hyphen
|
|
222
|
+
or unit.key in ENCLITICS
|
|
223
|
+
or (groups[-1][-1].stressed and groups[-1][-1].key in PROCLITICS)
|
|
224
|
+
)
|
|
225
|
+
):
|
|
226
|
+
groups[-1].append(unit)
|
|
227
|
+
else:
|
|
228
|
+
pending.append(unit)
|
|
229
|
+
if pending:
|
|
230
|
+
if groups:
|
|
231
|
+
groups[-1].extend(pending)
|
|
232
|
+
else:
|
|
233
|
+
groups.append(pending)
|
|
234
|
+
return groups
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _segments(units: Sequence[Unit]) -> list[Segment]:
|
|
238
|
+
segments: list[Segment] = []
|
|
239
|
+
for number, unit in enumerate(units):
|
|
240
|
+
text = respell(unit.text)
|
|
241
|
+
spelled = spell(text)
|
|
242
|
+
full = unit.key in NONREDUCING and not unit.hyphen
|
|
243
|
+
loan = _HARD_E.search(text) is not None
|
|
244
|
+
for segment in spelled:
|
|
245
|
+
segment.unit, segment.loan = number, loan
|
|
246
|
+
segment.full = segment.full or (full and segment.phone == "o")
|
|
247
|
+
first = spelled[0] if spelled else None
|
|
248
|
+
if (
|
|
249
|
+
first
|
|
250
|
+
and first.phone in VOWEL_SET
|
|
251
|
+
and segments
|
|
252
|
+
and segments[-1].phone not in VOWEL_SET
|
|
253
|
+
and segments[-1].phone not in SOFT_PHONES
|
|
254
|
+
): # к игре -> kɨɡrʲe, в огороде
|
|
255
|
+
first.context = "hard"
|
|
256
|
+
if first.phone == "i":
|
|
257
|
+
first.phone = "ɨ"
|
|
258
|
+
segments.extend(spelled)
|
|
259
|
+
return segments
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def soften(segments: list[Segment]) -> None:
|
|
263
|
+
"""Regressive palatalization (с сестрой, женщина, есть, песня)."""
|
|
264
|
+
for i in range(len(segments) - 2, -1, -1):
|
|
265
|
+
segment, following = segments[i], segments[i + 1]
|
|
266
|
+
if segment.phone not in SOFT:
|
|
267
|
+
continue
|
|
268
|
+
if following.phone == SOFT[segment.phone] or (
|
|
269
|
+
segment.unit == following.unit
|
|
270
|
+
and (
|
|
271
|
+
(segment.phone == "n" and following.phone in ("tɕ", "ɕ"))
|
|
272
|
+
or (
|
|
273
|
+
segment.phone in ("s", "z")
|
|
274
|
+
and following.phone in ("tʲ", "dʲ", "nʲ")
|
|
275
|
+
and not segment.initial
|
|
276
|
+
and not segment.loan
|
|
277
|
+
)
|
|
278
|
+
)
|
|
279
|
+
):
|
|
280
|
+
segment.phone = SOFT[segment.phone]
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def velar(segments: list[Segment]) -> None:
|
|
284
|
+
"""Soft/hard г before к or ч becomes х (легко, мягкий)."""
|
|
285
|
+
for segment, following in zip(segments, segments[1:], strict=False):
|
|
286
|
+
if (
|
|
287
|
+
segment.phone in ("ɡ", "ɡʲ")
|
|
288
|
+
and following.phone in ("k", "kʲ", "tɕ")
|
|
289
|
+
and segment.unit == following.unit
|
|
290
|
+
):
|
|
291
|
+
segment.phone = "x" if segment.phone == "ɡ" else "xʲ"
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def voicing(segments: list[Segment]) -> None:
|
|
295
|
+
"""Regressive voicing assimilation and word-final devoicing (в is transparent)."""
|
|
296
|
+
state: str | None = "voiceless" # word-final devoicing
|
|
297
|
+
for segment in reversed(segments):
|
|
298
|
+
if segment.phone not in OBSTRUENTS:
|
|
299
|
+
state = None
|
|
300
|
+
continue
|
|
301
|
+
if state == "voiceless":
|
|
302
|
+
segment.phone = VOICELESS_PARTNER.get(segment.phone, segment.phone)
|
|
303
|
+
elif state == "voiced":
|
|
304
|
+
segment.phone = VOICED_PARTNER.get(segment.phone, segment.phone)
|
|
305
|
+
if segment.letter != "в":
|
|
306
|
+
state = "voiceless" if segment.phone in VOICELESS else "voiced"
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def merge(segments: list[Segment]) -> list[Segment]:
|
|
310
|
+
"""Collapse geminates and assimilated sibilant/affricate clusters into single tokens."""
|
|
311
|
+
out: list[Segment] = []
|
|
312
|
+
for segment in segments:
|
|
313
|
+
previous = out[-1].phone if out else None
|
|
314
|
+
phone = segment.phone
|
|
315
|
+
if previous is None or previous in VOWEL_SET or phone in VOWEL_SET:
|
|
316
|
+
out.append(segment)
|
|
317
|
+
elif phone == previous:
|
|
318
|
+
continue
|
|
319
|
+
elif (
|
|
320
|
+
(previous in ("s", "sʲ", "ʂ") and phone in ("tɕ", "ɕ"))
|
|
321
|
+
or (previous in ("s", "sʲ") and phone == "ʂ")
|
|
322
|
+
or (previous in ("z", "zʲ") and phone == "ʐ")
|
|
323
|
+
or (previous in ("t", "tʲ", "d", "dʲ") and phone in ("ts", "tɕ"))
|
|
324
|
+
):
|
|
325
|
+
out[-1].phone = "ɕ" if phone == "tɕ" and previous in ("s", "sʲ", "ʂ") else phone
|
|
326
|
+
else:
|
|
327
|
+
out.append(segment)
|
|
328
|
+
return out
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def reduce(segments: list[Segment]) -> None:
|
|
332
|
+
"""Apply Russian unstressed vowel reduction relative to the stressed vowel."""
|
|
333
|
+
vowels = [i for i, s in enumerate(segments) if s.phone in VOWEL_SET]
|
|
334
|
+
stress = next((i for i in vowels if segments[i].stressed), None)
|
|
335
|
+
if stress is None:
|
|
336
|
+
return # phrase of clitics only
|
|
337
|
+
for k, i in enumerate(vowels):
|
|
338
|
+
segment = segments[i]
|
|
339
|
+
if segment.stressed or segment.full:
|
|
340
|
+
continue
|
|
341
|
+
if i > stress:
|
|
342
|
+
position = "post"
|
|
343
|
+
elif k + 1 < len(vowels) and vowels[k + 1] == stress:
|
|
344
|
+
position = "pre1"
|
|
345
|
+
else:
|
|
346
|
+
position = "pre"
|
|
347
|
+
following = segments[i + 1].phone if i + 1 < len(segments) else ""
|
|
348
|
+
segment.phone = _reduced(segment, position, following in SOFT_PHONES)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _reduced(segment: Segment, position: str, before_soft: bool) -> str:
|
|
352
|
+
quality, context = segment.phone, segment.context
|
|
353
|
+
if quality in ("u", "i", "ɨ"):
|
|
354
|
+
return quality
|
|
355
|
+
if context == "soft":
|
|
356
|
+
# ə in final syllable (тётя, вишням), i before soft consonant (память)
|
|
357
|
+
return (
|
|
358
|
+
"ə"
|
|
359
|
+
if quality == "a" and position == "post" and segment.last and not before_soft
|
|
360
|
+
else "i"
|
|
361
|
+
)
|
|
362
|
+
if quality == "e": # hard е in loanwords stays e (интернет)
|
|
363
|
+
return {"": "i", "hard": "e"}.get(context, "ɨ")
|
|
364
|
+
if position == "pre1" or (position == "pre" and context == ""):
|
|
365
|
+
return "ɐ"
|
|
366
|
+
return "ə"
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def phrase_phones(words: Sequence[str]) -> list[list[tuple[str, int]]]:
|
|
370
|
+
"""Convert stressed words of one phrase into phonetic words ``[(phone, stress)]``."""
|
|
371
|
+
out: list[list[tuple[str, int]]] = []
|
|
372
|
+
for group in _groups([unit for word in words for unit in _units(word)]):
|
|
373
|
+
segments = _segments(group)
|
|
374
|
+
velar(segments)
|
|
375
|
+
voicing(segments)
|
|
376
|
+
soften(segments)
|
|
377
|
+
segments = merge(segments)
|
|
378
|
+
reduce(segments)
|
|
379
|
+
if segments:
|
|
380
|
+
out.append([(s.phone, int(s.stressed)) for s in segments])
|
|
381
|
+
return out
|