ru-normalizr 0.1.0__tar.gz

ru_normalizr-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 NickZaitsev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ru_normalizr-0.1.0/MANIFEST.in

@@ -0,0 +1,9 @@
+include LICENSE
+include README.md
+include pyproject.toml
+recursive-include dictionaries *.dic
+recursive-include numerals *.py
+prune tests
+prune scripts
+prune examples
+global-exclude __pycache__ *.py[cod] *.pkl

ru_normalizr-0.1.0/PKG-INFO

@@ -0,0 +1,228 @@
+Metadata-Version: 2.4
+Name: ru-normalizr
+Version: 0.1.0
+Summary: Normalization-only Russian text preprocessing.
+Author: NickZaitsev
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/NickZaitsev/ru-normalizr
+Project-URL: Repository, https://github.com/NickZaitsev/ru-normalizr
+Project-URL: Issues, https://github.com/NickZaitsev/ru-normalizr/issues
+Project-URL: Changelog, https://github.com/NickZaitsev/ru-normalizr/blob/main/CHANGELOG.md
+Keywords: russian,normalization,nlp,text-processing,tts
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: Russian
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: eng_to_ipa==0.0.2
+Requires-Dist: num2words>=0.5.13
+Requires-Dist: pymorphy3>=2.0
+Requires-Dist: pymorphy3-dicts-ru>=2.0
+Requires-Dist: roman>=4.0
+Dynamic: license-file
+
+# ru-normalizr
+
+Normalization-only Russian text preprocessing extracted into a standalone package.
+
+`ru-normalizr` focuses on deterministic Russian text normalization:
+- years, dates, time, decimals, fractions, ordinals, and cardinal numerals
+- abbreviations, initials, Roman numerals, cleanup rules, and glued OCR-like text
+- Latin transliteration via dictionary rules or `eng_to_ipa`
+
+Out of scope by design:
+- accentization and stress dictionaries
+- pronunciation and post-phoneme fixes
+- TTS pause hacks and chunking
+- audio, model, or engine integration
+
+## Installation
+
+```bash
+pip install ru-normalizr
+```
+
+`eng_to_ipa` is installed by default, so the IPA backend is available out of the box.
+
+## API
+
+```python
+from ru_normalizr import NormalizeOptions, Normalizer, normalize, preprocess_text
+
+text = normalize("Глава IV. Встреча в 10:07.")
+prepared = preprocess_text("10кг")
+
+tts_normalizer = Normalizer(NormalizeOptions.tts())
+batch = tts_normalizer.normalize_batch(["Глава IV.", "В 1980-е годы было 25 млн."])
+roman_only = tts_normalizer.run_stage("roman", "Глава IV")
+```
+
+### Example outputs
+
+```python
+from ru_normalizr import normalize
+
+print(normalize("Глава IV. Встреча в 10:07."))
+# Глава четыре. Встреча в десять, ноль семь.
+
+print(normalize("В 1980-е годы было 25 млн. $"))
+# В тысяча девятьсот восьмидесятые годы было двадцать пять миллионов долларов
+
+print(normalize("Добавьте 1/4 стакана воды."))
+# Добавьте одну четвертую стакана воды.
+
+print(normalize("И. О. Фамилия приехал."))
+# и о фамилия приехал.
+```
+
+### Modes
+
+`NormalizeOptions()` uses the conservative `safe` preset by default.
+Use `NormalizeOptions.tts()` when you want the more aggressive TTS-oriented behavior.
+
+`safe` is intended for general text where it is more important not to over-normalize:
+- keeps all-caps headings as-is
+- keeps initials like `И. О. Фамилия`
+- keeps letter abbreviations like `ГИБДД`
+- keeps bracketed numeric links like `(1)` and `[1]`
+- keeps Latin transliteration disabled
+
+`tts` is intended for speech-oriented pipelines:
+- enables caps normalization and first-word decap
+- removes confident bracketed numeric links
+- expands initials and letter-by-letter abbreviations
+- enables Latin transliteration
+- keeps IPA stress markers disabled by default unless explicitly requested
+
+### Configuring options
+
+```python
+from ru_normalizr import NormalizeOptions, normalize
+
+options = NormalizeOptions.tts(
+    latinization_backend="ipa",
+    enable_latinization_stress_marks=False,
+)
+
+print(normalize("YouTube в 2024 г.", options))
+```
+
+You can also start from the conservative preset and override individual flags:
+
+```python
+from ru_normalizr import NormalizeOptions, normalize
+
+options = NormalizeOptions.safe(
+    enable_letter_abbreviation_expansion=True,
+    enable_latinization=True,
+    latinization_backend="dictionary",
+)
+
+print(normalize("USB drive", options))
+```
+
+Granular abbreviation controls:
+- `enable_contextual_abbreviation_expansion` for contextual abbreviations such as `т. д.`, `т. п.`, `млн.`, `тыс.`
+- `enable_initials_expansion` for patterns such as `И. О. Фамилия`
+- `enable_letter_abbreviation_expansion` for letter-by-letter expansions such as `ГИБДД`, `ООН`, `USB`
+
+Latinization controls:
+- `enable_latinization`
+- `latinization_backend="ipa" | "dictionary"`
+- `enable_latinization_stress_marks`
+
+When `latinization_backend="ipa"`, stress markers are omitted by default.
+Enable `enable_latinization_stress_marks=True` if you want `+` markers in the output.
+
+### Example dictionaries
+
+Runtime dictionary assets shipped in the package live under `ru_normalizr/dictionaries/`.
+Latinization rules live under `ru_normalizr/dictionaries/latinization/`.
+The source tree also includes `dictionaries/your_dictionary.dic` as an editable example dictionary for local customization.
+That example file is included in source distributions and excluded from published wheels.
+
+### Batch usage
+
+```python
+from ru_normalizr import Normalizer
+
+normalizer = Normalizer(NormalizeOptions.tts())
+texts = ["Глава IV.", "12.03.2025", "Цена 1.5 кг сахара."]
+print(normalizer.normalize_batch(texts))
+```
+
+Available stage names for expert use:
+- `preprocess`
+- `roman`
+- `years`
+- `dates_time`
+- `numerals`
+- `abbreviations`
+- `dictionary`
+- `latinization`
+- `finalize`
+
+Stage order is fixed in the main pipeline. Stage-level calls are for debugging, testing, and focused use, not for arbitrary reordering.
+
+## CLI
+
+```bash
+python -m ru_normalizr "Глава IV. Встреча в 10:07."
+echo "В 1980-е годы было 25 млн." | python -m ru_normalizr
+ru-normalizr --mode safe "ГИБДД"
+ru-normalizr --mode tts --file ./sample.txt
+ru-normalizr --mode tts --file ./sample.txt --output ./sample.normalized.txt
+ru-normalizr --mode tts --latinization-backend ipa --with-latin-stress "YouTube в 2024 г."
+```
+
+Useful CLI flags:
+- `--mode safe|tts`
+- `--latinization-backend ipa|dictionary`
+- `--with-latin-stress`
+- `--no-latinization`
+- `--no-first-word-decap`
+- `--keep-links`
+
+## Development
+
+```bash
+py -3.12 -m pip install -r ./ru_normalizr/requirements-dev.txt
+py -3.12 ./ru_normalizr/scripts/dev.py test
+py -3.12 ./ru_normalizr/scripts/dev.py lint
+py -3.12 ./ru_normalizr/scripts/dev.py build
+```
+
+## Release Notes
+
+- Changelog: `CHANGELOG.md`
+- Versioning policy: `VERSIONING.md`
+- Publish checklist: `PYPI_RELEASE_CHECKLIST.md`
+
+## Packaging
+
+The package is self-contained inside `ru_normalizr/` and builds as a standalone wheel from that directory:
+
+```bash
+python -m pip wheel --no-deps ./ru_normalizr
+```
+
+For repeatable local workflows, use the helper script:
+
+```bash
+py -3.12 ./ru_normalizr/scripts/dev.py clean
+py -3.12 ./ru_normalizr/scripts/dev.py test
+py -3.12 ./ru_normalizr/scripts/dev.py lint
+py -3.12 ./ru_normalizr/scripts/dev.py build
+```
+
+The supported public Python imports are from the package root, for example:
+
+```python
+from ru_normalizr import NormalizeOptions, Normalizer, normalize, preprocess_text
+```

ru_normalizr-0.1.0/README.md

@@ -0,0 +1,199 @@
+# ru-normalizr
+
+Normalization-only Russian text preprocessing extracted into a standalone package.
+
+`ru-normalizr` focuses on deterministic Russian text normalization:
+- years, dates, time, decimals, fractions, ordinals, and cardinal numerals
+- abbreviations, initials, Roman numerals, cleanup rules, and glued OCR-like text
+- Latin transliteration via dictionary rules or `eng_to_ipa`
+
+Out of scope by design:
+- accentization and stress dictionaries
+- pronunciation and post-phoneme fixes
+- TTS pause hacks and chunking
+- audio, model, or engine integration
+
+## Installation
+
+```bash
+pip install ru-normalizr
+```
+
+`eng_to_ipa` is installed by default, so the IPA backend is available out of the box.
+
+## API
+
+```python
+from ru_normalizr import NormalizeOptions, Normalizer, normalize, preprocess_text
+
+text = normalize("Глава IV. Встреча в 10:07.")
+prepared = preprocess_text("10кг")
+
+tts_normalizer = Normalizer(NormalizeOptions.tts())
+batch = tts_normalizer.normalize_batch(["Глава IV.", "В 1980-е годы было 25 млн."])
+roman_only = tts_normalizer.run_stage("roman", "Глава IV")
+```
+
+### Example outputs
+
+```python
+from ru_normalizr import normalize
+
+print(normalize("Глава IV. Встреча в 10:07."))
+# Глава четыре. Встреча в десять, ноль семь.
+
+print(normalize("В 1980-е годы было 25 млн. $"))
+# В тысяча девятьсот восьмидесятые годы было двадцать пять миллионов долларов
+
+print(normalize("Добавьте 1/4 стакана воды."))
+# Добавьте одну четвертую стакана воды.
+
+print(normalize("И. О. Фамилия приехал."))
+# и о фамилия приехал.
+```
+
+### Modes
+
+`NormalizeOptions()` uses the conservative `safe` preset by default.
+Use `NormalizeOptions.tts()` when you want the more aggressive TTS-oriented behavior.
+
+`safe` is intended for general text where it is more important not to over-normalize:
+- keeps all-caps headings as-is
+- keeps initials like `И. О. Фамилия`
+- keeps letter abbreviations like `ГИБДД`
+- keeps bracketed numeric links like `(1)` and `[1]`
+- keeps Latin transliteration disabled
+
+`tts` is intended for speech-oriented pipelines:
+- enables caps normalization and first-word decap
+- removes confident bracketed numeric links
+- expands initials and letter-by-letter abbreviations
+- enables Latin transliteration
+- keeps IPA stress markers disabled by default unless explicitly requested
+
+### Configuring options
+
+```python
+from ru_normalizr import NormalizeOptions, normalize
+
+options = NormalizeOptions.tts(
+    latinization_backend="ipa",
+    enable_latinization_stress_marks=False,
+)
+
+print(normalize("YouTube в 2024 г.", options))
+```
+
+You can also start from the conservative preset and override individual flags:
+
+```python
+from ru_normalizr import NormalizeOptions, normalize
+
+options = NormalizeOptions.safe(
+    enable_letter_abbreviation_expansion=True,
+    enable_latinization=True,
+    latinization_backend="dictionary",
+)
+
+print(normalize("USB drive", options))
+```
+
+Granular abbreviation controls:
+- `enable_contextual_abbreviation_expansion` for contextual abbreviations such as `т. д.`, `т. п.`, `млн.`, `тыс.`
+- `enable_initials_expansion` for patterns such as `И. О. Фамилия`
+- `enable_letter_abbreviation_expansion` for letter-by-letter expansions such as `ГИБДД`, `ООН`, `USB`
+
+Latinization controls:
+- `enable_latinization`
+- `latinization_backend="ipa" | "dictionary"`
+- `enable_latinization_stress_marks`
+
+When `latinization_backend="ipa"`, stress markers are omitted by default.
+Enable `enable_latinization_stress_marks=True` if you want `+` markers in the output.
+
+### Example dictionaries
+
+Runtime dictionary assets shipped in the package live under `ru_normalizr/dictionaries/`.
+Latinization rules live under `ru_normalizr/dictionaries/latinization/`.
+The source tree also includes `dictionaries/your_dictionary.dic` as an editable example dictionary for local customization.
+That example file is included in source distributions and excluded from published wheels.
+
+### Batch usage
+
+```python
+from ru_normalizr import Normalizer
+
+normalizer = Normalizer(NormalizeOptions.tts())
+texts = ["Глава IV.", "12.03.2025", "Цена 1.5 кг сахара."]
+print(normalizer.normalize_batch(texts))
+```
+
+Available stage names for expert use:
+- `preprocess`
+- `roman`
+- `years`
+- `dates_time`
+- `numerals`
+- `abbreviations`
+- `dictionary`
+- `latinization`
+- `finalize`
+
+Stage order is fixed in the main pipeline. Stage-level calls are for debugging, testing, and focused use, not for arbitrary reordering.
+
+## CLI
+
+```bash
+python -m ru_normalizr "Глава IV. Встреча в 10:07."
+echo "В 1980-е годы было 25 млн." | python -m ru_normalizr
+ru-normalizr --mode safe "ГИБДД"
+ru-normalizr --mode tts --file ./sample.txt
+ru-normalizr --mode tts --file ./sample.txt --output ./sample.normalized.txt
+ru-normalizr --mode tts --latinization-backend ipa --with-latin-stress "YouTube в 2024 г."
+```
+
+Useful CLI flags:
+- `--mode safe|tts`
+- `--latinization-backend ipa|dictionary`
+- `--with-latin-stress`
+- `--no-latinization`
+- `--no-first-word-decap`
+- `--keep-links`
+
+## Development
+
+```bash
+py -3.12 -m pip install -r ./ru_normalizr/requirements-dev.txt
+py -3.12 ./ru_normalizr/scripts/dev.py test
+py -3.12 ./ru_normalizr/scripts/dev.py lint
+py -3.12 ./ru_normalizr/scripts/dev.py build
+```
+
+## Release Notes
+
+- Changelog: `CHANGELOG.md`
+- Versioning policy: `VERSIONING.md`
+- Publish checklist: `PYPI_RELEASE_CHECKLIST.md`
+
+## Packaging
+
+The package is self-contained inside `ru_normalizr/` and builds as a standalone wheel from that directory:
+
+```bash
+python -m pip wheel --no-deps ./ru_normalizr
+```
+
+For repeatable local workflows, use the helper script:
+
+```bash
+py -3.12 ./ru_normalizr/scripts/dev.py clean
+py -3.12 ./ru_normalizr/scripts/dev.py test
+py -3.12 ./ru_normalizr/scripts/dev.py lint
+py -3.12 ./ru_normalizr/scripts/dev.py build
+```
+
+The supported public Python imports are from the package root, for example:
+
+```python
+from ru_normalizr import NormalizeOptions, Normalizer, normalize, preprocess_text
+```

ru_normalizr-0.1.0/__init__.py

@@ -0,0 +1,7 @@
+"""Russian text normalization library."""
+
+from .options import NormalizeOptions
+from .pipeline import Normalizer, normalize, preprocess_text
+
+__all__ = ["NormalizeOptions", "Normalizer", "normalize", "preprocess_text"]
+__version__ = "0.1.0"

ru_normalizr-0.1.0/__main__.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .options import NormalizeOptions
+from .pipeline import normalize
+
+
+def _read_input(args: argparse.Namespace) -> str:
+    if args.file:
+        return Path(args.file).read_text(encoding="utf-8")
+    if args.text:
+        return args.text
+    if not sys.stdin.isatty():
+        return sys.stdin.read()
+    raise SystemExit("Provide text, --file, or stdin.")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="ru-normalizr", description="Normalize Russian text."
+    )
+    parser.add_argument("text", nargs="?", help="Inline text to normalize.")
+    parser.add_argument("--file", help="Read text from file.")
+    parser.add_argument(
+        "--output", help="Write normalized text to file instead of stdout."
+    )
+    parser.add_argument(
+        "--check", action="store_true", help="Normalize input and print the result."
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["safe", "tts"],
+        default="safe",
+        help="Preset normalization mode.",
+    )
+    parser.add_argument(
+        "--latinization-backend",
+        choices=["ipa", "dictionary"],
+        help="Backend for Latin transliteration.",
+    )
+    parser.add_argument(
+        "--no-latinization", action="store_true", help="Disable Latin transliteration."
+    )
+    parser.add_argument(
+        "--no-first-word-decap", action="store_true", help="Disable first-word decap."
+    )
+    parser.add_argument(
+        "--keep-links", action="store_true", help="Keep bracketed numeric links."
+    )
+    parser.add_argument(
+        "--with-latin-stress",
+        action="store_true",
+        help="Keep '+' stress markers when using IPA latinization.",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    text = _read_input(args)
+    options = NormalizeOptions(
+        mode=args.mode,
+        enable_first_word_decap=False if args.no_first_word_decap else None,
+        remove_links=False if args.keep_links else None,
+        enable_latinization=False if args.no_latinization else None,
+        latinization_backend=args.latinization_backend,
+        enable_latinization_stress_marks=args.with_latin_stress,
+    )
+    result = normalize(text, options)
+    if args.output:
+        output_path = Path(args.output)
+        output_text = result if result.endswith("\n") else result + "\n"
+        output_path.write_text(output_text, encoding="utf-8")
+    else:
+        sys.stdout.write(result)
+        if not result.endswith("\n"):
+            sys.stdout.write("\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ru_normalizr-0.1.0/_morph.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+import functools
+
+import pymorphy3
+
+
+@functools.lru_cache(maxsize=1)
+def get_morph() -> pymorphy3.MorphAnalyzer:
+    return pymorphy3.MorphAnalyzer()

ru_normalizr-0.1.0/abbreviation_rules.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class RegexRule:
+    pattern: str
+    flags: int = re.IGNORECASE
+
+
+def rx(pattern: str, flags: int = re.IGNORECASE) -> RegexRule:
+    return RegexRule(pattern=pattern, flags=flags)
+
+
+ABBREVIATION_RULES = {
+    rx(r"\bсм\.\s*рис\.(?!\w)"): "смотри рисунок",
+    rx(r"\bсм\."): "смотри",
+    rx(r"\bнапр\."): "например",
+    rx(r"\bт\.?\s*е\."): "то есть",
+    rx(r"\bт\.?\s*к\."): "так как",
+    rx(r"(?<!\w)н\.?\s*э\.(?!\w)"): "нашей эры",
+    rx(r"(?<!\w)нэ\.?(?!\w)"): "нашей эры",
+    rx(r"\bс\.\s*(?=\d)"): "страница ",
+    rx(r"\bp\.\s*(?=\d)"): "страница ",
+    rx(r"\bст\.\s*(?=\d)"): "статья ",
+    rx(r"\bгл\."): "глава",
+    rx(r"\bг-?н\.?\b"): "господин",
+    rx(r"\bг-?жа\.?\b"): "госпожа",
+    rx(r"\bрис\.\s*(?=\d)"): "рисунок ",
+    rx(r"\bим\."): "имени",
+    "снск.": "сноска",
+    "прим. пер.": "примечание переводчика",
+    "прим.пер.": "примечание переводчика",
+    "прим. перев.": "примечание переводчика",
+    "прим.перев.": "примечание переводчика",
+    "прим. ред.": "примечание редактора",
+    "прим.ред.": "примечание редактора",
+    rx(r"\bперев\."): "переводчик",
+    "англ.": "английский",
+    "руск.": "русский",
+    "рус.": "русский",
+    "немецк.": "немецкий",
+    "нем.": "немецкий",
+    "франц.": "французский",
+    "греч.": "греческий",
+    "латин.": "латинский",
+    "лат.": "латинский",
+    "изд-во": "издательство",
+    "ин-т": "институт",
+    "инф-ция": "информация",
+    "орг-ция": "организация",
+    "мин-во": "министерство",
+    "гос-во": "государство",
+    "гос.": "государственный",
+    "междунар.": "международный",
+    "обл.": "область",
+    "р-н": "район",
+    "пос.": "посёлок",
+    "пгт": "посёлок городского типа",
+    "оз.": "озеро",
+    "полит.": "политический",
+    "офиц.": "официальный",
+    "юридич.": "юридический",
+    "ж.-д.": "железнодорожный",
+    "с.-х.": "сельскохозяйственный",
+    "штаб-кв.": "штаб-квартира",
+    "т.д.": "так далее",
+    "т. д.": "так далее",
+    "т.п.": "тому подобное",
+    "т. п.": "тому подобное",
+    "табл.": "таблица",
+    "стр.": "страница",
+    "ред.": "редактор",
+    "сост.": "составитель",
+    "прил.": "приложение",
+    "прим. авт.": "примечание автора",
+    "прим.": "примечание",
+    "УК РФ": "уголовный кодекс российской федерации",
+    "АК РФ": "арбитражный кодекс российской федерации",
+    "псевд.": "псевдоним",
+    "респ.": "республика",
+}
+
+
+def _compile_literal_rule(abbr: str) -> re.Pattern[str]:
+    return re.compile(rf"(?<!\w){re.escape(abbr)}(?!\w)", re.IGNORECASE)
+
+
+ABBREVIATION_PATTERNS = [
+    (
+        (
+            re.compile(matcher.pattern, matcher.flags)
+            if isinstance(matcher, RegexRule)
+            else _compile_literal_rule(matcher)
+        ),
+        replacement,
+    )
+    for matcher, replacement in ABBREVIATION_RULES.items()
+]