jayron 0.1.0__py3-none-any.whl

jayron/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+# Free for non-commercial use only; modification and redistribution are not allowed.
+# Commercial use requires a license. See the LICENSE file or contact r.raxmonberdiyev10@gmail.com.
+
+"""jayron — tools for Uzbek speech recognition."""
+
+__version__ = "0.1.0"

jayron/normalizers/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+# Free for non-commercial use only; modification and redistribution are not allowed.
+# Commercial use requires a license. See the LICENSE file or contact r.raxmonberdiyev10@gmail.com.
+
+"""Text normalizers.
+
+    from jayron.normalizers import uz_normilizer, converter
+    uz_normilizer("2024-yil 5 ta kitob")   # 'ikki ming yigirma toʻrtinchi yil beshta kitob'
+    converter("Bog'chaga bordingmi?")      # 'Boğçaga bordingmi?'  (new 2026 alphabet)
+"""
+
+from .uz import AlphabetConverter, UzbekNormalizer, converter, uz_normalizer, uz_normilizer
+
+__all__ = ["AlphabetConverter", "UzbekNormalizer", "converter", "uz_normalizer", "uz_normilizer"]

jayron/normalizers/uz/__init__.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+# Free for non-commercial use only; modification and redistribution are not allowed.
+# Commercial use requires a license. See the LICENSE file or contact r.raxmonberdiyev10@gmail.com.
+
+"""Uzbek text normalization for ASR."""
+
+from .converter import AlphabetConverter, converter, to_new, to_old
+from .normalizer import OKINA, UzbekNormalizer, expand_numbers
+from .numbers import number_to_ordinal, number_to_words
+from .transliterate import cyrillic_to_latin
+
+uz_normalizer = UzbekNormalizer()
+uz_normilizer = uz_normalizer  # same object, under the spelling used in the project
+
+__all__ = [
+    "OKINA",
+    "AlphabetConverter",
+    "UzbekNormalizer",
+    "converter",
+    "cyrillic_to_latin",
+    "to_new",
+    "to_old",
+    "expand_numbers",
+    "number_to_ordinal",
+    "number_to_words",
+    "uz_normalizer",
+    "uz_normilizer",
+]

jayron/normalizers/uz/__main__.py

@@ -0,0 +1,75 @@
+# Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+# Free for non-commercial use only; modification and redistribution are not allowed.
+# Commercial use requires a license. See the LICENSE file or contact r.raxmonberdiyev10@gmail.com.
+
+"""Command line: normalize text, text files, or JSONL manifests.
+
+    python -m jayron.normalizers.uz "2024-yil 5 ta kitob"
+    python -m jayron.normalizers.uz -i corpus.txt -o corpus.norm.txt
+    python -m jayron.normalizers.uz -i train.jsonl -o train.norm.jsonl --jsonl --field text
+    cat corpus.txt | python -m jayron.normalizers.uz
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import List, Optional
+
+from .normalizer import OKINA, UzbekNormalizer
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    p = argparse.ArgumentParser(prog="jayron-uz-norm", description="Uzbek text normalizer for ASR")
+    p.add_argument("text", nargs="*", help="text to normalize (otherwise reads --input or stdin)")
+    p.add_argument("-i", "--input", help="input file (one sentence per line, or JSONL with --jsonl)")
+    p.add_argument("-o", "--output", help="output file (default: stdout)")
+    p.add_argument("--jsonl", action="store_true", help="input is JSON lines (e.g. a NeMo manifest)")
+    p.add_argument("--field", default="text", help="JSONL field to normalize (default: text)")
+    p.add_argument("--out-field", help="write the result to this field instead of overwriting --field")
+    p.add_argument("--alphabet", choices=("old", "new"), default="old",
+                   help="old: oʻ gʻ sh ch (default); new: ö ğ ş ç (2026 reform)")
+    p.add_argument("--apostrophe", default=OKINA, help="character for oʻ/gʻ/tutuq (default: ʻ U+02BB)")
+    p.add_argument("--keep-case", action="store_true")
+    p.add_argument("--keep-punctuation", action="store_true")
+    p.add_argument("--no-transliterate", action="store_true", help="leave Cyrillic text as is")
+    p.add_argument("--no-numbers", action="store_true", help="leave digits as is")
+    args = p.parse_args(argv)
+
+    norm = UzbekNormalizer(
+        alphabet=args.alphabet,
+        apostrophe=args.apostrophe,
+        lowercase=not args.keep_case,
+        remove_punctuation=not args.keep_punctuation,
+        transliterate=not args.no_transliterate,
+        expand_numbers=not args.no_numbers,
+    )
+
+    if args.text:
+        print(norm(" ".join(args.text)))
+        return 0
+
+    src = open(args.input, encoding="utf-8") if args.input else sys.stdin
+    dst = open(args.output, "w", encoding="utf-8") if args.output else sys.stdout
+    try:
+        for line in src:
+            line = line.rstrip("\n")
+            if args.jsonl:
+                if not line.strip():
+                    continue
+                record = json.loads(line)
+                record[args.out_field or args.field] = norm(record.get(args.field) or "")
+                dst.write(json.dumps(record, ensure_ascii=False) + "\n")
+            else:
+                dst.write(norm(line) + "\n")
+    finally:
+        if src is not sys.stdin:
+            src.close()
+        if dst is not sys.stdout:
+            dst.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

jayron/normalizers/uz/converter.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+# Free for non-commercial use only; modification and redistribution are not allowed.
+# Commercial use requires a license. See the LICENSE file or contact r.raxmonberdiyev10@gmail.com.
+
+"""Convert Uzbek text between the 1995 Latin alphabet and the 2026 reform.
+
+The reform (law signed 21 Oct 2026, in force from 22 Apr 2027) replaces four
+letters and drops Ng as a separate letter:
+
+    Oʻ -> Ö    Gʻ -> Ğ    Sh -> Ş    Ch -> Ç    (ng stays written as n + g)
+
+The apostrophe (tutuq belgisi, as in maʼno) is kept, and so is the s'h spelling
+of a separate s + h (Is'hoq stays Is'hoq).
+
+    >>> from jayron.normalizers import converter
+    >>> converter("Bog'chaga bordingmi?")
+    'Boğçaga bordingmi?'
+    >>> converter.to_old("Boğçaga bordingmi?")
+    'Bogʻchaga bordingmi?'
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+from .numbers import OKINA
+from .transliterate import cyrillic_to_latin
+
+__all__ = ["APOSTROPHES", "AlphabetConverter", "NEW_LETTERS", "converter", "to_new", "to_old"]
+
+# ' ` ´ ʹ ʻ ʼ ʽ ‘ ’ ‛ ′  — all the ways people type the oʻ / gʻ mark
+APOSTROPHES = "['`´ʹʻʼʽ‘’‛′]"
+NEW_LETTERS = "öğşçÖĞŞÇ"
+
+_OG = re.compile("([oOgG])" + APOSTROPHES)
+_DIGRAPH = re.compile("([sScC])[hH]")
+_TO_NEW = {"o": "ö", "O": "Ö", "g": "ğ", "G": "Ğ", "s": "ş", "S": "Ş", "c": "ç", "C": "Ç"}
+
+# New letters, plus look-alikes from drafts and other keyboards (ō from the 2021 draft, ḡ, ș, ǧ ...).
+_OLD_OG = {"ö": "o", "ō": "o", "ğ": "g", "ḡ": "g", "ǵ": "g", "ǧ": "g"}
+_OLD_OG.update({k.upper(): v.upper() for k, v in list(_OLD_OG.items())})
+_OLD_DIGRAPH = {"ş": "s", "ș": "s", "ç": "c"}
+_OLD_DIGRAPH.update({k.upper(): v.upper() for k, v in list(_OLD_DIGRAPH.items())})
+_NEW_LETTER = re.compile("[" + "".join(_OLD_OG) + "".join(_OLD_DIGRAPH) + "]")
+
+# Links and e-mail addresses are left untouched.
+_PROTECTED = re.compile(r"(?:https?://|www\.)\S+|[\w.+-]+@[\w-]+(?:\.[\w-]+)+")
+
+
+def _letters_to_old(text: str, apostrophe: str = OKINA) -> str:
+    """ö -> oʻ, ğ -> gʻ, ş -> sh, ç -> ch (also accepts ō, ḡ, ș, ...)."""
+    if text.isascii():
+        return text
+
+    def repl(m: "re.Match[str]") -> str:
+        ch = m.group()
+        if ch in _OLD_OG:
+            return _OLD_OG[ch] + apostrophe
+        base = _OLD_DIGRAPH[ch]
+        if base.islower():
+            return base + "h"
+        i, s = m.start(), m.string
+        prev = s[i - 1] if i else ""
+        nxt = s[i + 1] if i + 1 < len(s) else ""
+        all_caps = nxt.isupper() or (prev.isupper() and not nxt.isalpha())
+        return base + ("H" if all_caps else "h")
+
+    return _NEW_LETTER.sub(repl, text)
+
+
+def _letters_to_new(text: str) -> str:
+    text = _OG.sub(lambda m: _TO_NEW[m.group(1)], text)
+    return _DIGRAPH.sub(lambda m: _TO_NEW[m.group(1)], text)
+
+
+def _outside_links(text: str, fn) -> str:
+    parts, last = [], 0
+    for m in _PROTECTED.finditer(text):
+        parts.append(fn(text[last:m.start()]))
+        parts.append(m.group())
+        last = m.end()
+    parts.append(fn(text[last:]))
+    return "".join(parts)
+
+
+def to_new(text: str, *, transliterate: bool = True) -> str:
+    """Convert to the new alphabet: "Bog'chaga" -> "Boğçaga".
+
+    Accepts the old Latin alphabet (any apostrophe variant), Cyrillic (when
+    ``transliterate`` is on), or text already partly in the new alphabet.
+    Everything else (case, punctuation, digits, ng, tutuq) is kept as is.
+    """
+    if not text:
+        return text
+    text = unicodedata.normalize("NFC", text)
+    if transliterate:
+        text = cyrillic_to_latin(text)
+    return _outside_links(text, lambda part: _letters_to_new(_letters_to_old(part)))
+
+
+def to_old(text: str, *, apostrophe: str = OKINA, transliterate: bool = True) -> str:
+    """Convert to the 1995 alphabet: "Boğçaga" -> "Bogʻchaga" (``apostrophe`` marks oʻ/gʻ)."""
+    if not text:
+        return text
+    text = unicodedata.normalize("NFC", text)
+    if transliterate:
+        text = cyrillic_to_latin(text)
+    return _letters_to_old(text, apostrophe)
+
+
+class AlphabetConverter:
+    """Old (1995) <-> new (2026) Uzbek Latin alphabet. Calling it converts to the new one.
+
+    Args:
+        transliterate: also accept Uzbek Cyrillic input.
+        apostrophe: character for oʻ/gʻ when converting back to the old alphabet.
+    """
+
+    def __init__(self, *, transliterate: bool = True, apostrophe: str = OKINA) -> None:
+        self.transliterate = transliterate
+        self.apostrophe = apostrophe
+
+    def to_new(self, text: str) -> str:
+        return to_new(text, transliterate=self.transliterate)
+
+    def to_old(self, text: str) -> str:
+        return to_old(text, apostrophe=self.apostrophe, transliterate=self.transliterate)
+
+    def __call__(self, text):
+        if isinstance(text, str):
+            return self.to_new(text)
+        return [self.to_new(t) for t in text]
+
+    def __repr__(self) -> str:
+        return f"AlphabetConverter(transliterate={self.transliterate!r}, apostrophe={self.apostrophe!r})"
+
+
+converter = AlphabetConverter()

jayron/normalizers/uz/normalizer.py

@@ -0,0 +1,484 @@
+# Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+# Free for non-commercial use only; modification and redistribution are not allowed.
+# Commercial use requires a license. See the LICENSE file or contact r.raxmonberdiyev10@gmail.com.
+
+"""Uzbek text normalizer for speech recognition (ASR).
+
+Turns written Uzbek into the form a speaker would say it: Latin script,
+numbers/dates/times/units spelled out, one apostrophe character, no
+punctuation, lowercase. Every step can be switched off.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from typing import Iterable, List, Optional, Sequence, Tuple, Union
+
+from .numbers import (
+    MONTHS,
+    OKINA,
+    attach_suffix,
+    decimal_to_words,
+    fraction_to_words,
+    number_to_ordinal,
+    read_integer,
+    roman_to_int,
+)
+from .converter import APOSTROPHES, _letters_to_old, to_new
+from .transliterate import cyrillic_to_latin, fix_mixed_script
+
+__all__ = ["UzbekNormalizer", "expand_numbers", "OKINA"]
+
+
+def _ok(text: str) -> str:
+    return text.replace("'", OKINA)
+
+
+# --------------------------------------------------------------------------- unicode clean-up
+
+_INVISIBLE = re.compile("[­​-‍⁠]")
+_SPACES = re.compile("[\t\n\r\f\v   - 

  　]")
+_COMPAT = re.compile("[ff-st！-～]")  # ligatures, full-width ASCII
+_NON_ASCII_DIGIT = re.compile(r"(?![0-9])\d")
+_HYPHENS = re.compile("[‐-‒−﹣]")
+_DASH_IN_RANGE = re.compile(r"(?<=\d) ?[–—] ?(?=\d)")
+_DASH_AS_MINUS = re.compile(r"(?<!\S)[–—](?=\d)")
+_ANNOTATION = re.compile(r"\[[^\[\]]{1,50}\]|</?[A-Za-z_][\w-]{0,30}>")
+_SUPERSCRIPTS = str.maketrans("⁰¹²³⁴⁵⁶⁷⁸⁹", "0123456789")
+
+
+def _pre_clean(text: str) -> str:
+    text = unicodedata.normalize("NFC", text)
+    text = _INVISIBLE.sub("", text)
+    text = _SPACES.sub(" ", text)
+    if not text.isascii():
+        text = _COMPAT.sub(lambda m: unicodedata.normalize("NFKC", m.group()), text)
+        text = _NON_ASCII_DIGIT.sub(lambda m: str(unicodedata.decimal(m.group())), text)
+        text = _HYPHENS.sub("-", text)
+        text = _DASH_IN_RANGE.sub("-", text)
+        text = _DASH_AS_MINUS.sub("-", text)
+    return text
+
+
+# --------------------------------------------------------------------------- apostrophes
+
+_APOS = APOSTROPHES
+_OG_APOS = re.compile("([oOgG])" + _APOS)
+_INNER_APOS = re.compile(r"(?<=[^\W\d_])" + _APOS + r"(?=[^\W\d_])")
+_STRAY_APOS = re.compile("(?!" + OKINA + ")" + _APOS)
+
+
+def _canonical_apostrophes(text: str) -> str:
+    """oʻ/gʻ and the tutuq belgisi all become ʻ; anything else apostrophe-like is a quote mark."""
+    text = _OG_APOS.sub(lambda m: m.group(1) + OKINA, text)
+    text = _INNER_APOS.sub(OKINA, text)
+    return _STRAY_APOS.sub("'", text)
+
+
+# --------------------------------------------------------------------------- abbreviations & symbols
+
+_NOT_AFTER_LETTER = r"(?<![^\W\d_])"
+_ABBREVIATIONS = [
+    (re.compile(r"(?<!\w)va\s+h\.\s?k\.?(?!\w)"), "va hokazo"),
+    (re.compile(r"(?<!\w)h\.\s?k\.?(?!\w)"), "hokazo"),
+    (re.compile(r"(?<!\w)va\s+b\.(?!\w)"), "va boshqalar"),
+    (re.compile(_NOT_AFTER_LETTER + r"mlrd\.?(?!\w)"), "milliard"),
+    (re.compile(_NOT_AFTER_LETTER + r"mln\.?(?!\w)"), "million"),
+    (re.compile(_NOT_AFTER_LETTER + r"trln\.?(?!\w)"), "trillion"),
+    (re.compile(r"(?<!\w)prof\.(?!\w)"), "professor"),
+    (re.compile(r"(?<!\w)dots\.(?!\w)"), "dotsent"),
+    (re.compile(r"(?<!\w)akad\.(?!\w)"), "akademik"),
+    (re.compile(r"(?<!\w)mas\.(?!\w)"), "masalan"),
+    (re.compile(_ok(r"(?<!\w)ko'ch\.(?!\w)")), _ok("ko'chasi")),
+    # "Toshkent sh.", "Samarqand v.", "Chilonzor t."
+    (re.compile(r"(?<!\w)([A-Z]\w+)\s+sh\.(?!\w)"), r"\1 shahri"),
+    (re.compile(r"(?<!\w)([A-Z]\w+)\s+v\.(?!\w)"), r"\1 viloyati"),
+    (re.compile(r"(?<!\w)([A-Z]\w+)\s+t\.(?!\w)"), r"\1 tumani"),
+]
+
+_SYMBOLS = {
+    "%": "foiz", "‰": "promille", "№": "raqam", "&": "va", "§": "paragraf", "°": "gradus",
+    "$": "dollar", "€": "yevro", "£": "funt sterling", "₽": "rubl", "¥": "iyena", "₸": "tenge",
+}
+_SYMBOL_RE = re.compile("[" + re.escape("".join(_SYMBOLS)) + "]")
+
+
+def _expand_abbreviations(text: str) -> str:
+    for pattern, replacement in _ABBREVIATIONS:
+        text = pattern.sub(replacement, text)
+    return text
+
+
+def _expand_symbols(text: str) -> str:
+    return _SYMBOL_RE.sub(lambda m: " " + _SYMBOLS[m.group()] + " ", text)
+
+
+# --------------------------------------------------------------------------- number helpers
+
+# A number as written: 25 | 1 000 000 | 1,000,000 | 1.000.000 | 3,5 | 2.75
+_NUM = (
+    r"(?<!\d)(?:"
+    r"[1-9]\d{0,2}(?:[ ,]\d{3})+(?!\d)"
+    r"|[1-9]\d{0,2}(?:\.\d{3}){2,}(?!\d)"
+    r"|\d+"
+    r")(?:[.,]\d+(?![.,]?\d))?"
+)
+_GROUPED = re.compile(r"([1-9]\d{0,2}(?:([ ,.])\d{3})+)(?:[.,](\d+))?")
+_PLAIN = re.compile(r"(\d+)(?:[.,](\d+))?")
+
+
+def _split_number(token: str) -> Tuple[str, Optional[str]]:
+    """'1 000,5' -> ('1000', '5');  '3.5' -> ('3', '5');  '1,500' -> ('1500', None)."""
+    m = _GROUPED.fullmatch(token)
+    # a single dot group ("1.500") is a decimal, not thousands
+    if m and (m.group(2) != "." or m.group(1).count(".") >= 2):
+        return re.sub(r"\D", "", m.group(1)), m.group(3)
+    m = _PLAIN.fullmatch(token)
+    if m:
+        return m.group(1), m.group(2)
+    return re.sub(r"\D", "", token), None
+
+
+def _say(token: str, ordinal: bool = False) -> str:
+    integer, fraction = _split_number(token)
+    if fraction is not None:
+        return decimal_to_words(integer, fraction)
+    if ordinal:
+        return number_to_ordinal(int(integer))
+    return read_integer(integer)
+
+
+def _say_range(first: Optional[str], second: str, ordinal: bool = False) -> str:
+    words = _say(second, ordinal)
+    return _say(first, ordinal) + " " + words if first else words
+
+
+_CASE = r"(?:dan|da|gacha|qacha|kacha|ga|qa|ka|ning|ni|dagi|dek|day|cha)"
+# glued to a cardinal: 5ta, 5-ta, 10dan, 40ga, 100lab, 5tasi
+_ATTACH = re.compile(rf"(?:ta|tasi|lar|lari|lik|lab)?{_CASE}?")
+# written as a separate word but still a suffix: "5 ta", "10 dan"
+_SPACED_ATTACH = re.compile(rf"ta(?:si)?{_CASE}?|{_CASE}")
+# ordinal markers: 5-chi, 2-inchi, 1-nchisi, may oyining 5-si
+_ORD_MARKER = re.compile(rf"(?:i?n)?chi((?:lar)?(?:i|si)?{_CASE}?)|(si{_CASE}?)")
+# after a hyphen these words take a cardinal: 3-xonali, 100-yillik, 30-yilligi
+_CARDINAL_HINT = re.compile(r"(?:li|lik|liq)$|lig")
+_MONTH = re.compile(r"(?:%s)(?:da|dan|ga|gacha|ning|ni|dagi|gi)?" % "|".join(MONTHS[1:]))
+
+
+def _verbalize(token: str, sep: Optional[str], word: Optional[str]) -> Tuple[str, bool]:
+    """Spell a number given the word that follows it. Returns (text, became_ordinal)."""
+    integer, fraction = _split_number(token)
+    if word is None:
+        return _say(token), False
+    low = word.lower()
+    if fraction is None:
+        n = int(integer)
+        if sep in ("", "-"):
+            m = _ORD_MARKER.fullmatch(low)
+            if m:
+                rest = m.group(1) if m.group(1) is not None else m.group(2)
+                return number_to_ordinal(n) + word[len(word) - len(rest):], True
+        if sep == " " and (
+            (1 <= n <= 31 and _MONTH.fullmatch(low))  # "15 mart"
+            or (1000 <= n <= 2999 and low.startswith("yil") and not _CARDINAL_HINT.search(low))  # "2024 yil"
+        ):
+            return number_to_ordinal(n) + " " + word, True
+    cardinal = _say(token)
+    if (sep in ("", "-") and _ATTACH.fullmatch(low)) or (sep == " " and _SPACED_ATTACH.fullmatch(low)):
+        return attach_suffix(cardinal, word), False
+    if sep == "-" and fraction is None and not _CARDINAL_HINT.search(low):
+        return number_to_ordinal(int(integer)) + " " + word, True  # 2024-yil, 5-sinf, 1-may
+    return cardinal + " " + word, False
+
+
+# --------------------------------------------------------------------------- number passes
+
+_PHONES = (
+    re.compile(r"(?<![\w+])\+?(998)[ -]?\(?(\d{2})\)?[ -]?(\d{3})[ -]?(\d{2})[ -]?(\d{2})(?!\d)"),
+    re.compile(r"(?<![\w+])\+(\d{1,3})[ -]?\(?(\d{2,4})\)?[ -]?(\d{2,4})[ -]?(\d{2})[ -]?(\d{2})(?!\d)"),
+    re.compile(r"(?<![\w+])\(?(\d{2})\)?[ -](\d{3})[ -](\d{2})[ -](\d{2})(?!\d)"),
+)
+_DATE_DMY = re.compile(
+    r"(?<![\d.,])(0?[1-9]|[12]\d|3[01])([./-])(0?[1-9]|1[0-2])\2((?:1[6-9]|20|21)\d{2})(?!\d)"
+    r"(?: ?-? ?(y\.|yil\w*))?"
+)
+_DATE_YMD = re.compile(r"(?<![\d.,-])((?:1[6-9]|20|21)\d{2})([-./])(0[1-9]|1[0-2])\2(0[1-9]|[12]\d|3[01])(?!\d)")
+_TIME_SUFFIX = r"(?:[- ](?=(?:da|dan|ga|gacha|cha|ni|ning|dagi)(?!\w)))?"
+_TIME = re.compile(r"(?<![\d:.,])([01]?\d|2[0-3]):([0-5]\d)(?::([0-5]\d))?(?![\d:])" + _TIME_SUFFIX)
+_SOAT_TIME = re.compile(r"(?<!\w)([Ss]oat\s+)([01]?\d|2[0-3])\.([0-5]\d)(?![\d.])" + _TIME_SUFFIX)
+_ROMAN_RE = re.compile(
+    r"(?<![\w-])(?=[MDCLXVI])(M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3}))(?: ?- ?| +)"
+    + _ok(r"(?=(?i:asr|bob|jild|qism|kurs|chorak|bosqich|modda|bo'lim|sinf|toifa|guruh|qurultoy))")
+)
+_VULGAR = {
+    "½": (1, 2), "⅓": (1, 3), "⅔": (2, 3), "¼": (1, 4), "¾": (3, 4), "⅕": (1, 5), "⅖": (2, 5),
+    "⅗": (3, 5), "⅘": (4, 5), "⅙": (1, 6), "⅚": (5, 6), "⅛": (1, 8), "⅜": (3, 8), "⅝": (5, 8), "⅞": (7, 8),
+}
+_VULGAR_RE = re.compile(r"(?<![\w.,])(?:(\d+) ?)?([" + "".join(_VULGAR) + "])")
+_FRACTION = re.compile(r"(?<![\w.,/])(\d{1,3})/(\d{1,3})(?![\d/])")
+_PLUS = re.compile(r"(?<=\d) ?\+ ?(?=\d)")
+_EQUALS = re.compile(r"(?<=\d) ?= ?(?=[-+]?\d)")
+_TIMES = re.compile(r"(?<=\d) ?[x×] ?(?=\d)")
+_MINUS_SIGN = re.compile(r"(?<![\w.,])-(?=\d)")
+_PLUS_SIGN = re.compile(r"(?<![\w.,])\+(?=\d)")
+
+_RANGE_PREFIX = rf"(?:({_NUM})-)?"  # "5-10%" -> both numbers belong to the unit
+# lets a unit word take a following suffix: "25%-ga", "25% ga" -> "foizga"
+_GLUE = rf"(?:-(?=[^\W\d_])| (?={_CASE}(?!\w)))?"
+_SCALE = r"(?: ?(ming|million|milliard|trillion)(?!\w))?"
+_CURRENCY_SYMBOLS = {"$": "dollar", "€": "yevro", "£": "funt sterling", "₽": "rubl", "¥": "iyena", "₸": "tenge"}
+_CURRENCY_CODES = {
+    "USD": "dollar", "EUR": "yevro", "RUB": "rubl", "UZS": _ok("so'm"), "KZT": "tenge",
+    "GBP": "funt sterling", "CNY": "yuan",
+}
+_SYM_CLASS = "[" + re.escape("".join(_CURRENCY_SYMBOLS)) + "]"
+_CURRENCY_BEFORE = re.compile(rf"({_SYM_CLASS}) ?({_NUM})(?:-({_NUM}))?{_SCALE}")
+_CURRENCY_AFTER = re.compile(rf"{_RANGE_PREFIX}({_NUM}){_SCALE} ?({_SYM_CLASS}){_GLUE}")
+_CURRENCY_CODE = re.compile(rf"{_RANGE_PREFIX}({_NUM}){_SCALE} ?({'|'.join(_CURRENCY_CODES)})(?![A-Za-z]){_GLUE}")
+_PERCENT = re.compile(rf"{_RANGE_PREFIX}({_NUM}) ?([%‰]){_GLUE}")
+_DEGREES = re.compile(rf"{_RANGE_PREFIX}({_NUM}) ?° ?([CSF](?![A-Za-z]))?{_GLUE}")
+
+_UNITS = {
+    "km²": "kvadrat kilometr", "m²": "kvadrat metr", "sm²": "kvadrat santimetr", "mm²": "kvadrat millimetr",
+    "km³": "kub kilometr", "m³": "kub metr", "sm³": "kub santimetr",
+    "km": "kilometr", "m": "metr", "sm": "santimetr", "cm": "santimetr", "mm": "millimetr",
+    "kg": "kilogramm", "g": "gramm", "gr": "gramm", "mg": "milligramm", "t": "tonna",
+    "l": "litr", "ml": "millilitr",
+    "kVt": "kilovatt", "MVt": "megavatt", "Vt": "vatt",
+    "KB": "kilobayt", "MB": "megabayt", "GB": "gigabayt", "TB": "terabayt",
+    "min": "daqiqa", "sek": "soniya",
+}
+_UNIT_RE = re.compile(
+    rf"{_RANGE_PREFIX}({_NUM}) ?({'|'.join(sorted(map(re.escape, _UNITS), key=len, reverse=True))})(?!\w){_GLUE}"
+)
+_YEAR_ABBR = re.compile(r"(?<![\w.,])(?:(\d{3,4})-)?(\d{3,4}) ?-? ?(yy|y)\.(?!\w)")
+_RANGE = re.compile(rf"(?<![\w.,])({_NUM})-({_NUM})(?:(-| ?)([^\W\d_]\w*))?")
+_NUMBER_WORD = re.compile(rf"(?<![\w.,])({_NUM})(?:(-| ?)([^\W\d_]\w*))?")
+_ANY_DIGITS = re.compile(r"\d+")
+
+
+def _date_dmy(m: "re.Match[str]") -> str:
+    day, _, month, year, year_word = m.groups()
+    year_word = "yil" if year_word in (None, "y.") else year_word
+    return f"{number_to_ordinal(int(day))} {MONTHS[int(month)]} {number_to_ordinal(int(year))} {year_word}"
+
+
+def _date_ymd(m: "re.Match[str]") -> str:
+    year, _, month, day = m.groups()
+    return f"{number_to_ordinal(int(year))} yil {number_to_ordinal(int(day))} {MONTHS[int(month)]}"
+
+
+def _time(hours: str, minutes: str, seconds: Optional[str] = None) -> str:
+    parts = [read_integer(hours.lstrip("0") or "0"), read_integer(minutes)]
+    if seconds:
+        parts.append(read_integer(seconds))
+    return " ".join(parts)
+
+
+def _vulgar(m: "re.Match[str]") -> str:
+    numerator, denominator = _VULGAR[m.group(2)]
+    words = fraction_to_words(numerator, denominator)
+    return (_say(m.group(1)) + " butun " + words) if m.group(1) else words
+
+
+def _fraction(m: "re.Match[str]") -> str:
+    numerator, denominator = int(m.group(1)), int(m.group(2))
+    if 0 < numerator < denominator:
+        return fraction_to_words(numerator, denominator)
+    return m.group()
+
+
+def _with_scale(text: str, scale: Optional[str]) -> str:
+    return f"{text} {scale}" if scale else text
+
+
+def _range(m: "re.Match[str]") -> str:
+    first, second, sep, word = m.groups()
+    second_text, ordinal = _verbalize(second, sep, word)
+    integer, fraction = _split_number(first)
+    first_text = number_to_ordinal(int(integer)) if ordinal and fraction is None else _say(first)
+    return first_text + " " + second_text
+
+
+def expand_numbers(text: str) -> str:
+    """Spell out every number in ``text`` (expects canonical apostrophes)."""
+    text = _ROMAN_RE.sub(lambda m: number_to_ordinal(roman_to_int(m.group(1))) + " ", text)
+    if not any(c.isdigit() or c in _VULGAR for c in text):
+        return text
+    for pattern in _PHONES:
+        text = pattern.sub(lambda m: " ".join(read_integer(g) for g in m.groups()), text)
+    text = _DATE_DMY.sub(_date_dmy, text)
+    text = _DATE_YMD.sub(_date_ymd, text)
+    text = _TIME.sub(lambda m: _time(*m.groups()), text)
+    text = _SOAT_TIME.sub(lambda m: m.group(1) + _time(m.group(2), m.group(3)), text)
+    text = _VULGAR_RE.sub(_vulgar, text)
+    text = _FRACTION.sub(_fraction, text)
+    text = _PLUS.sub(" plyus ", text)
+    text = _EQUALS.sub(" teng ", text)
+    text = _TIMES.sub(" karra ", text)
+    text = _MINUS_SIGN.sub("minus ", text)
+    text = _PLUS_SIGN.sub("plyus ", text)
+    text = _CURRENCY_BEFORE.sub(
+        lambda m: _with_scale(_say_range(m.group(2), m.group(3)) if m.group(3) else _say(m.group(2)), m.group(4))
+        + " " + _CURRENCY_SYMBOLS[m.group(1)],
+        text,
+    )
+    text = _CURRENCY_AFTER.sub(
+        lambda m: _with_scale(_say_range(m.group(1), m.group(2)), m.group(3)) + " " + _CURRENCY_SYMBOLS[m.group(4)],
+        text,
+    )
+    text = _CURRENCY_CODE.sub(
+        lambda m: _with_scale(_say_range(m.group(1), m.group(2)), m.group(3)) + " " + _CURRENCY_CODES[m.group(4)],
+        text,
+    )
+    text = _PERCENT.sub(
+        lambda m: _say_range(m.group(1), m.group(2)) + (" foiz" if m.group(3) == "%" else " promille"), text
+    )
+    text = _DEGREES.sub(
+        lambda m: _say_range(m.group(1), m.group(2)) + " gradus" + (" farengeyt" if m.group(3) == "F" else ""),
+        text,
+    )
+    text = _UNIT_RE.sub(lambda m: _say_range(m.group(1), m.group(2)) + " " + _UNITS[m.group(3)], text)
+    text = _YEAR_ABBR.sub(
+        lambda m: _say_range(m.group(1), m.group(2), ordinal=True) + (" yillar" if m.group(3) == "yy" else " yil"),
+        text,
+    )
+    text = _RANGE.sub(_range, text)
+    text = _NUMBER_WORD.sub(lambda m: _verbalize(*m.groups())[0], text)
+    # anything left (digits glued to letters: "A5", superscripts) is read plainly
+    text = text.translate(_SUPERSCRIPTS)
+    return _ANY_DIGITS.sub(lambda m: " " + read_integer(m.group()) + " ", text)
+
+
+# --------------------------------------------------------------------------- final clean-up
+
+_LATIN_SPECIAL = str.maketrans({"ı": "i", "ø": "o", "Ø": "O", "ł": "l", "Ł": "L", "đ": "d", "Đ": "D", "ß": "ss"})
+
+
+def _strip_latin_accents(text: str) -> str:
+    """é -> e, ö -> o, ç -> c. Cyrillic letters (й, ё) are left alone."""
+    if text.isascii():
+        return text
+    text = text.translate(_LATIN_SPECIAL)
+    out = []
+    after_latin = False
+    for ch in unicodedata.normalize("NFD", text):
+        if unicodedata.combining(ch):
+            if after_latin:
+                continue
+        else:
+            after_latin = ch.isascii() and ch.isalpha()
+        out.append(ch)
+    return unicodedata.normalize("NFC", "".join(out))
+
+
+_NON_WORD = re.compile(r"[^\w\s]|_")
+_LONE_OKINA = re.compile(r"(?<![^\W\d_])" + OKINA)
+_SPACE_BEFORE_PUNCT = re.compile(r" +([.,!?;:)\]»])")
+_MULTISPACE = re.compile(r"\s+")
+
+Rule = Tuple[Union[str, "re.Pattern[str]"], str]
+
+
+class UzbekNormalizer:
+    """Normalize Uzbek text for ASR training and evaluation.
+
+    >>> norm = UzbekNormalizer()
+    >>> norm("2024-yil 15-mart kuni soat 14:30 da 1,5 mln so'm to'landi.")
+    'ikki ming yigirma toʻrtinchi yil oʻn beshinchi mart kuni soat oʻn toʻrt oʻttizda bir butun oʻndan besh million soʻm toʻlandi'
+
+    Args:
+        transliterate: convert Cyrillic Uzbek to Latin.
+        expand_numbers: spell out numbers, dates, times, money, percent, units.
+        expand_abbreviations: expand abbreviations (mln, h.k., sh.) and symbols (%, №, &).
+        lowercase: lowercase the output.
+        remove_punctuation: drop punctuation and quote marks (word-internal apostrophes stay).
+        strip_accents: é -> e, ü -> u for foreign Latin letters.
+        drop_annotations: remove [noise], [1], <unk> style tags.
+        apostrophe: the single character used for oʻ, gʻ and the tutuq belgisi.
+            Default is ʻ (U+02BB); pass "'" for plain ASCII.
+        custom_rules: extra (regex, replacement) pairs applied first, for domain terms.
+        alphabet: "old" (1995: oʻ gʻ sh ch) or "new" (2026 reform: ö ğ ş ç).
+            Input in either alphabet is accepted.
+    """
+
+    def __init__(
+        self,
+        *,
+        transliterate: bool = True,
+        expand_numbers: bool = True,
+        expand_abbreviations: bool = True,
+        lowercase: bool = True,
+        remove_punctuation: bool = True,
+        strip_accents: bool = True,
+        drop_annotations: bool = True,
+        apostrophe: str = OKINA,
+        custom_rules: Sequence[Rule] = (),
+        alphabet: str = "old",
+    ) -> None:
+        if alphabet not in ("old", "new"):
+            raise ValueError(f'alphabet must be "old" or "new", got {alphabet!r}')
+        self.alphabet = alphabet
+        self.transliterate = transliterate
+        self.expand_numbers = expand_numbers
+        self.expand_abbreviations = expand_abbreviations
+        self.lowercase = lowercase
+        self.remove_punctuation = remove_punctuation
+        self.strip_accents = strip_accents
+        self.drop_annotations = drop_annotations
+        self.apostrophe = apostrophe
+        self.custom_rules = [(re.compile(p) if isinstance(p, str) else p, r) for p, r in custom_rules]
+
+    def normalize(self, text: str) -> str:
+        if not text:
+            return ""
+        text = _pre_clean(text)
+        for pattern, replacement in self.custom_rules:
+            text = pattern.sub(replacement, text)
+        if self.drop_annotations:
+            text = _ANNOTATION.sub(" ", text)
+        text = fix_mixed_script(text)
+        if self.transliterate:
+            text = cyrillic_to_latin(text)
+        text = _canonical_apostrophes(text)
+        # work internally in the old alphabet: ö ğ ş ç -> oʻ gʻ sh ch
+        text = _letters_to_old(text)
+        if self.expand_abbreviations:
+            text = _expand_abbreviations(text)
+        if self.expand_numbers:
+            text = expand_numbers(text)
+        if self.expand_abbreviations:
+            text = _expand_symbols(text)
+        if self.lowercase:
+            text = text.lower()
+        if self.strip_accents:
+            text = _strip_latin_accents(text)
+        if self.remove_punctuation:
+            text = _NON_WORD.sub(" ", text)
+            text = _LONE_OKINA.sub("", text)
+            text = _MULTISPACE.sub(" ", text).strip()
+        else:
+            text = _MULTISPACE.sub(" ", text).strip()
+            text = _SPACE_BEFORE_PUNCT.sub(r"\1", text)
+        if self.alphabet == "new":
+            text = to_new(text, transliterate=False)
+        if self.apostrophe != OKINA:
+            text = text.replace(OKINA, self.apostrophe)
+        return text
+
+    def normalize_batch(self, texts: Iterable[str]) -> List[str]:
+        return [self.normalize(t) for t in texts]
+
+    def __call__(self, text: Union[str, Iterable[str]]) -> Union[str, List[str]]:
+        if isinstance(text, str):
+            return self.normalize(text)
+        return self.normalize_batch(text)
+
+    def __repr__(self) -> str:
+        opts = ", ".join(
+            f"{k}={getattr(self, k)!r}"
+            for k in ("alphabet", "transliterate", "expand_numbers", "expand_abbreviations", "lowercase",
+                      "remove_punctuation", "strip_accents", "drop_annotations", "apostrophe")
+        )
+        return f"UzbekNormalizer({opts})"

jayron/normalizers/uz/numbers.py

@@ -0,0 +1,133 @@
+# Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+# Free for non-commercial use only; modification and redistribution are not allowed.
+# Commercial use requires a license. See the LICENSE file or contact r.raxmonberdiyev10@gmail.com.
+
+"""Spell numbers as Uzbek (Latin script) words."""
+
+from __future__ import annotations
+
+from typing import Tuple
+
+OKINA = "ʻ"  # ʻ  (as in oʻ, gʻ)
+
+
+def _ok(*words: str) -> Tuple[str, ...]:
+    """Write words with ASCII ' in the source, store them with the okina."""
+    return tuple(w.replace("'", OKINA) for w in words)
+
+
+ONES = _ok("nol", "bir", "ikki", "uch", "to'rt", "besh", "olti", "yetti", "sakkiz", "to'qqiz")
+TENS = _ok("", "o'n", "yigirma", "o'ttiz", "qirq", "ellik", "oltmish", "yetmish", "sakson", "to'qson")
+MONTHS = (
+    "", "yanvar", "fevral", "mart", "aprel", "may", "iyun",
+    "iyul", "avgust", "sentabr", "oktabr", "noyabr", "dekabr",
+)
+
+_SCALES = ((10 ** 12, "trillion"), (10 ** 9, "milliard"), (10 ** 6, "million"), (10 ** 3, "ming"))
+_LIMIT = 10 ** 15  # from here on, numbers are read digit by digit
+_SCALE_WORDS = frozenset(("yuz", "ming", "million", "milliard", "trillion"))
+_DECIMAL_PLACES = dict(enumerate(_ok("o'ndan", "yuzdan", "mingdan", "o'n mingdan", "yuz mingdan", "milliondan"), 1))
+_VOWELS = frozenset("aeiou")
+_ROMAN = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
+
+
+def _below_thousand(n: int, *, bir_yuz: bool) -> str:
+    hundreds, rest = divmod(n, 100)
+    parts = []
+    if hundreds == 1:
+        parts.append("bir yuz" if bir_yuz else "yuz")
+    elif hundreds:
+        parts.append(ONES[hundreds] + " yuz")
+    tens, ones = divmod(rest, 10)
+    if tens:
+        parts.append(TENS[tens])
+    if ones:
+        parts.append(ONES[ones])
+    return " ".join(parts)
+
+
+def digits_to_words(digits: str) -> str:
+    """'007' -> 'nol nol yetti'."""
+    return " ".join(ONES[int(d)] for d in digits if d in "0123456789")
+
+
+def number_to_words(n: int) -> str:
+    """Cardinal number: 2024 -> 'ikki ming yigirma toʻrt'."""
+    n = int(n)
+    if n < 0:
+        return "minus " + number_to_words(-n)
+    if n == 0:
+        return ONES[0]
+    if n >= _LIMIT:
+        return digits_to_words(str(n))
+    parts = []
+    for value, name in _SCALES:
+        count, n = divmod(n, value)
+        if count == 1:
+            parts.append("bir " + name)
+        elif count:
+            # a hundred that multiplies a scale is bare: 100 000 -> "yuz ming"
+            parts.append(_below_thousand(count, bir_yuz=False) + " " + name)
+    if n:
+        parts.append(_below_thousand(n, bir_yuz=True))
+    return " ".join(parts)
+
+
+def to_ordinal(words: str) -> str:
+    """Turn spelled cardinal words into the ordinal form (suffix on the last word)."""
+    parts = words.split()
+    # 100 -> "yuzinchi", 1000 -> "minginchi" (not "bir yuzinchi")
+    if len(parts) == 2 and parts[0] == "bir" and parts[1] in _SCALE_WORDS:
+        parts = parts[1:]
+    last = parts[-1]
+    parts[-1] = last + ("nchi" if last[-1] in _VOWELS else "inchi")
+    return " ".join(parts)
+
+
+def number_to_ordinal(n: int) -> str:
+    """Ordinal number: 2 -> 'ikkinchi', 2024 -> 'ikki ming yigirma toʻrtinchi'."""
+    return to_ordinal(number_to_words(n))
+
+
+def read_integer(digits: str) -> str:
+    """Read a digit string; a leading zero means it is read digit by digit ('05' -> 'nol besh')."""
+    if len(digits) > 1 and digits[0] == "0":
+        return digits_to_words(digits)
+    return number_to_words(int(digits))
+
+
+def decimal_to_words(integer: str, fraction: str) -> str:
+    """'3', '5' -> 'uch butun oʻndan besh'; '2', '75' -> 'ikki butun yuzdan yetmish besh'."""
+    whole = number_to_words(int(integer))
+    place = _DECIMAL_PLACES.get(len(fraction))
+    if place is None:
+        return whole + " butun " + digits_to_words(fraction)
+    return whole + " butun " + place + " " + number_to_words(int(fraction))
+
+
+def attach_suffix(words: str, suffix: str) -> str:
+    """Glue a grammatical suffix to a number word: 'qirq' + 'ga' -> 'qirqqa'."""
+    if not suffix:
+        return words
+    if words.endswith("q") and suffix[0] in "gG":
+        suffix = "q" + suffix[1:]
+    elif words.endswith("k") and suffix[0] in "gG":
+        suffix = "k" + suffix[1:]
+    return words + suffix
+
+
+def fraction_to_words(numerator: int, denominator: int) -> str:
+    """3/4 -> 'toʻrtdan uch'."""
+    return attach_suffix(number_to_words(denominator), "dan") + " " + number_to_words(numerator)
+
+
+def roman_to_int(roman: str) -> int:
+    total = prev = 0
+    for ch in reversed(roman.upper()):
+        value = _ROMAN[ch]
+        if value < prev:
+            total -= value
+        else:
+            total += value
+            prev = value
+    return total

jayron/normalizers/uz/transliterate.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+# Free for non-commercial use only; modification and redistribution are not allowed.
+# Commercial use requires a license. See the LICENSE file or contact r.raxmonberdiyev10@gmail.com.
+
+"""Uzbek Cyrillic -> Latin transliteration and mixed-script (homoglyph) repair."""
+
+from __future__ import annotations
+
+import re
+
+from .numbers import OKINA
+
+TUTUQ = "ʼ"  # ʼ  (tutuq belgisi)
+
+_CYR_VOWELS = frozenset("аеёиоуўэюяы")
+
+_CYR2LAT = {
+    "а": "a", "б": "b", "в": "v", "г": "g", "д": "d", "ё": "yo", "ж": "j", "з": "z",
+    "и": "i", "й": "y", "к": "k", "л": "l", "м": "m", "н": "n", "о": "o", "п": "p",
+    "р": "r", "с": "s", "т": "t", "у": "u", "ф": "f", "х": "x", "ч": "ch", "ш": "sh",
+    "щ": "sh", "ъ": TUTUQ, "ы": "i", "ь": "", "э": "e", "ю": "yu", "я": "ya",
+    "ў": "o" + OKINA, "қ": "q", "ғ": "g" + OKINA, "ҳ": "h",
+    # letters that sneak in from Russian / Kazakh / Tajik keyboards
+    "ӯ": "o" + OKINA, "і": "i", "ї": "yi", "є": "ye", "ґ": "g", "ң": "ng",
+    "ә": "a", "ө": "o", "ү": "u", "һ": "h", "ҷ": "j",
+}
+_CONTEXT_LETTERS = frozenset("ецҳ")
+_HAS_CYRILLIC = re.compile("[Ѐ-ӿ]")
+
+
+def cyrillic_to_latin(text: str) -> str:
+    """Transliterate Uzbek Cyrillic to the official Latin alphabet, keeping case.
+
+    Rules beyond one-to-one letters:
+      * е -> "ye" at the start of a word or after a vowel/ъ/ь, otherwise "e"
+      * ц -> "ts" after a vowel, otherwise "s"
+      * сҳ -> "sʼh" (so it is not read as the digraph "sh")
+    """
+    if not _HAS_CYRILLIC.search(text):
+        return text
+    out = []
+    length = len(text)
+    for i, ch in enumerate(text):
+        low = ch.lower()
+        if low not in _CYR2LAT and low not in _CONTEXT_LETTERS:
+            out.append(ch)
+            continue
+        prev = text[i - 1] if i else ""
+        prev_low = prev.lower()
+        if low == "е":
+            starts_word = not prev.isalpha()
+            lat = "ye" if starts_word or prev_low in _CYR_VOWELS or prev_low in "ъь" else "e"
+        elif low == "ц":
+            lat = "ts" if prev_low in _CYR_VOWELS else "s"
+        elif low == "ҳ":
+            lat = TUTUQ + "h" if prev_low == "с" else "h"
+        else:
+            lat = _CYR2LAT[low]
+        if lat and ch.isupper():
+            nxt = text[i + 1] if i + 1 < length else ""
+            all_caps = nxt.isupper() or (prev.isupper() and not nxt.isalpha())
+            lat = lat.upper() if all_caps else lat[0].upper() + lat[1:]
+        out.append(lat)
+    return "".join(out)
+
+
+# Letters that look identical in Latin and Cyrillic.
+_CYR_LOOKALIKE = {
+    "а": "a", "е": "e", "о": "o", "р": "p", "с": "c", "у": "y", "х": "x", "і": "i",
+    "ј": "j", "ԛ": "q", "ѕ": "s", "һ": "h",
+    "А": "A", "В": "B", "Е": "E", "К": "K", "М": "M", "Н": "H", "О": "O", "Р": "P",
+    "С": "C", "Т": "T", "Х": "X", "У": "Y", "І": "I", "Ј": "J", "Ѕ": "S",
+}
+_TO_LATIN = str.maketrans(_CYR_LOOKALIKE)
+_TO_CYRILLIC = str.maketrans({lat: cyr for cyr, lat in _CYR_LOOKALIKE.items() if cyr not in "іјԛѕһІЈЅ"})
+_WORD = re.compile(r"[^\W\d_]+")
+
+
+def _repair_word(match: "re.Match[str]") -> str:
+    word = match.group()
+    cyrillic = sum(1 for c in word if "Ѐ" <= c <= "ӿ")
+    if not cyrillic or cyrillic == len(word):
+        return word
+    latin = sum(1 for c in word if "a" <= c.lower() <= "z")
+    if not latin:
+        return word
+    # Majority script wins: fix the look-alike letters of the minority script.
+    return word.translate(_TO_LATIN if latin >= cyrillic else _TO_CYRILLIC)
+
+
+def fix_mixed_script(text: str) -> str:
+    """Repair words that mix Latin and Cyrillic look-alikes, e.g. 'Toshkеnt' with a Cyrillic 'е'."""
+    if not _HAS_CYRILLIC.search(text):
+        return text
+    return _WORD.sub(_repair_word, text)

jayron-0.1.0.dist-info/METADATA

@@ -0,0 +1,170 @@
+Metadata-Version: 2.4
+Name: jayron
+Version: 0.1.0
+Summary: Uzbek text normalizer for speech recognition (ASR): numbers, dates, Cyrillic, apostrophes, new 2026 alphabet.
+Author: Rakhimberdi Rakhmonberdiev
+License-Expression: LicenseRef-jayron-NonCommercial
+Keywords: uzbek,asr,speech-recognition,text-normalization,wer,transliteration,nlp
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: license-file
+
+# jayron
+
+Tools for Uzbek speech recognition. The first module is a text normalizer that
+turns written Uzbek into the form a speaker says it, for ASR training targets
+and for scoring (WER/CER). No dependencies, Python 3.9+.
+
+> Free for non-commercial use. Commercial use requires a license, see [License](#license).
+
+```bash
+pip install jayron
+```
+
+```python
+from jayron.normalizers import uz_normilizer
+
+uz_normilizer("2024-yil 15-mart kuni soat 14:30 da 1,5 mln so'm to'landi.")
+# 'ikki ming yigirma toʻrtinchi yil oʻn beshinchi mart kuni soat oʻn toʻrt oʻttizda
+#  bir butun oʻndan besh million soʻm toʻlandi'
+
+uz_normilizer(["Ўзбекистон", "5 ta kitob"])   # lists work too
+# ['oʻzbekiston', 'beshta kitob']
+```
+
+`uz_normalizer` is the same object under the standard spelling.
+
+## What it does
+
+| Input | Output |
+|---|---|
+| `O‘zbekiston`, `O'zbekiston`, `O`zbekiston` | `oʻzbekiston` (one apostrophe character) |
+| `Ўзбекистон Республикаси` | `oʻzbekiston respublikasi` |
+| `Toshkеnt` (Cyrillic `е` inside) | `toshkent` |
+| `137`, `1 000 000`, `1,000,000` | `bir yuz oʻttiz yetti`, `bir million` |
+| `2024-yil`, `5-sinf`, `2024 yil`, `1991 y.` | `ikki ming yigirma toʻrtinchi yil`, `beshinchi sinf`, ... |
+| `5ta`, `5 ta`, `40ga`, `10dan` | `beshta`, `beshta`, `qirqqa`, `oʻndan` |
+| `3-xonali`, `30-yillik` | `uch xonali`, `oʻttiz yillik` (cardinal, not ordinal) |
+| `3,5`, `2.75` | `uch butun oʻndan besh`, `ikki butun yuzdan yetmish besh` |
+| `1/2`, `½`, `2½` | `ikkidan bir`, `ikkidan bir`, `ikki butun ikkidan bir` |
+| `25%`, `12,5%ga` | `yigirma besh foiz`, `oʻn ikki butun oʻndan besh foizga` |
+| `$100`, `50 €`, `100 USD`, `$1,5 mln` | `bir yuz dollar`, `ellik yevro`, ..., `bir butun oʻndan besh million dollar` |
+| `5 kg`, `50 m²`, `-5°C` | `besh kilogramm`, `ellik kvadrat metr`, `minus besh gradus` |
+| `14:30`, `soat 9:05da` | `oʻn toʻrt oʻttiz`, `soat toʻqqiz nol beshda` |
+| `12.05.2024`, `2024-05-12`, `15 mart` | `oʻn ikkinchi may ikki ming yigirma toʻrtinchi yil`, ..., `oʻn beshinchi mart` |
+| `5-10 kishi`, `1990-2000-yillar` | `besh oʻn kishi`, `... toʻqsoninchi ikki minginchi yillar` |
+| `XXI asr`, `IV bob` | `yigirma birinchi asr`, `toʻrtinchi bob` |
+| `+998 90 123 45 67` | read group by group |
+| `va h.k.`, `Toshkent sh.`, `№5`, `&` | `va hokazo`, `toshkent shahri`, `raqam besh`, `va` |
+| `[noise]`, `<unk>`, `[1]` | removed |
+| `Café`, `«Salom!»` | `cafe`, `salom` |
+
+With the defaults, output contains only `a-z`, `ʻ` and single spaces.
+
+## New alphabet (2026 reform)
+
+The reform law (signed 21 Oct 2026, in force from 22 Apr 2027) replaces
+`Oʻ Gʻ Sh Ch` with `Ö Ğ Ş Ç` and drops `Ng` as a separate letter (it stays
+written as n + g). The apostrophe is kept.
+
+```python
+from jayron.normalizers import converter
+
+converter("Bog'chaga bordingmi?")          # 'Boğçaga bordingmi?'
+converter("Ўзбекистон шаҳри")              # 'Özbekiston şahri'   (Cyrillic works too)
+converter.to_old("Boğçaga bordingmi?")     # 'Bogʻchaga bordingmi?'
+```
+
+Only those four letters change; case, punctuation, digits, `ng`, the tutuq
+(`maʼno`) and `s'h` (`Is'hoq`) are left as they are, and so are links and
+e-mail addresses.
+
+To get normalized ASR text in the new alphabet, use `UzbekNormalizer(alphabet="new")`:
+
+```python
+UzbekNormalizer(alphabet="new")("Bog'chaga 5 ta o'quvchi bordi.")
+# 'boğçaga beşta öquvçi bordi'
+```
+
+The normalizer reads either alphabet as input, so a corpus that mixes old
+and new spellings comes out consistent.
+
+## Options
+
+```python
+from jayron.normalizers import UzbekNormalizer
+
+norm = UzbekNormalizer(
+    alphabet="old",              # "new" -> ö ğ ş ç
+    apostrophe="'",              # default ʻ (U+02BB); "'" if your tokenizer prefers ASCII
+    lowercase=True,
+    remove_punctuation=True,     # False keeps punctuation (for punctuation-aware models)
+    transliterate=True,          # Cyrillic -> Latin
+    expand_numbers=True,
+    expand_abbreviations=True,   # mln, h.k., sh., %, №, &
+    strip_accents=True,          # é -> e
+    drop_annotations=True,       # [noise], <unk>
+    custom_rules=[(r"\bAI\b", "sun'iy intellekt")],  # your own regex rules, applied first
+)
+```
+
+Use the same settings for training transcripts and for scoring (both references
+and model output), or WER will count formatting differences as errors.
+
+Lower-level helpers live in `jayron.normalizers.uz`: `number_to_words`,
+`number_to_ordinal`, `cyrillic_to_latin`, `expand_numbers`.
+
+## Command line
+
+```bash
+jayron-uz-norm "2024-yil 5 ta kitob"
+jayron-uz-norm -i corpus.txt -o corpus.norm.txt
+jayron-uz-norm -i train.jsonl -o train.norm.jsonl --jsonl --field text   # NeMo manifest
+jayron-uz-norm --help
+```
+
+## Conventions and known ambiguities
+
+- Hundreds and thousands keep `bir` (`100` → `bir yuz`, `1000` → `bir ming`),
+  except when multiplying a larger scale (`100 000` → `yuz ming`) or in ordinals
+  (`100-yil` → `yuzinchi yil`).
+- `ʻ` is used for oʻ, gʻ and the tutuq belgisi alike, so the model has one
+  apostrophe token.
+- Space-grouped thousands are joined: `5 100` → `besh ming bir yuz`. In plain
+  text this is right almost always, but a list such as "5 100" would be merged.
+- A single dot group is a decimal: `1.500` → `bir butun mingdan besh yuz`;
+  a single comma group of three digits is thousands: `1,500` → `bir ming besh yuz`.
+- Times are read digit-pair by digit-pair: `14:00` → `oʻn toʻrt nol nol`.
+
+## Development
+
+```bash
+pip install -e '.[dev]'
+pytest
+```
+
+## License
+
+Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+
+jayron is **free for non-commercial use**: personal projects, study, teaching
+and non-profit research. **Commercial use** (companies, paid products or
+services, commercial ASR models or datasets, paid contract work) **needs a
+paid license**. Contact r.raxmonberdiyev10@gmail.com.
+
+The source code may not be modified, copied into other projects, or
+redistributed. See [LICENSE](LICENSE) for the full terms.

jayron-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+jayron/__init__.py,sha256=Od36H2Mn2ID2xUx7X9qsHx-DWspv7jkAQI5I3EnX61Q,331
+jayron/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+jayron/normalizers/__init__.py,sha256=LKq8zV2qhD1KoMqR-UvepenHINErhwSsMHnMaapV1QM,716
+jayron/normalizers/uz/__init__.py,sha256=CrstU7GY7BnUfpsJqhYS3pwxfAhbGOpfr45YcdU-SXU,906
+jayron/normalizers/uz/__main__.py,sha256=Basn_7a9oA-5RzTo2nyKLq7kIHhf_jhM6T86lAWr8D0,3223
+jayron/normalizers/uz/converter.py,sha256=5czB07ruKQEVIKlKAlAfWCE_VI6ZcXm8sfn6ED-u4mg,5135
+jayron/normalizers/uz/normalizer.py,sha256=XAujRt4hxp5qLrMuSGU_x8LHImObr2EJE_ckLPclTUI,20825
+jayron/normalizers/uz/numbers.py,sha256=U_p2VQ2nPDdMKM3YFc7THcImnpQ_icRNHMZgyhFuwNs,4714
+jayron/normalizers/uz/transliterate.py,sha256=mQoEJfhCu3gtFQbZ_tY2E2CiI7YR05eo0LN-2WhWYII,3873
+jayron-0.1.0.dist-info/licenses/LICENSE,sha256=TVwRhjWpCFjFSm90IaYGekUKHszqyjqhyb9umDt1T5A,3171
+jayron-0.1.0.dist-info/METADATA,sha256=W0WMZWMQl9WcFIMZ0tmQHwi4y5KQVXqkz6yrEhiuMhI,7051
+jayron-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+jayron-0.1.0.dist-info/entry_points.txt,sha256=SEj-GuxyMgc2A32LJQp3DxHGO1-7FKle760YGETFqu0,71
+jayron-0.1.0.dist-info/top_level.txt,sha256=E4YI7kcJnOO71QCTKVP6uZYBK6qTUrc7oxygax08iyQ,7
+jayron-0.1.0.dist-info/RECORD,,

jayron-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

jayron-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+jayron-uz-norm = jayron.normalizers.uz.__main__:main

jayron-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,71 @@
+jayron License (Non-Commercial, No Derivatives)
+
+Copyright (c) 2026 Rakhimberdi Rakhmonberdiev. All rights reserved.
+
+The jayron software, including its source code, word lists, rules, regular
+expressions and documentation (the "Software"), is source-available but is
+NOT open source. You may read the source code, but you may use it only as
+described below.
+
+1. FREE NON-COMMERCIAL USE
+
+   You may download, install and run the unmodified Software, and use the
+   text it produces (the "Output"), free of charge, for non-commercial
+   purposes only. Non-commercial purposes are:
+     (a) personal projects that earn no money;
+     (b) study and teaching;
+     (c) research at universities, schools or non-profit research
+         organizations, including research published in papers.
+
+   If you publish work that uses the Software, please credit
+   "jayron by Rakhimberdi Rakhmonberdiev".
+
+2. COMMERCIAL USE REQUIRES A PAID LICENSE
+
+   Any commercial use requires a separate written license from the
+   copyright holder. Commercial use includes, for example:
+     (a) any use by or for a company or other for-profit organization;
+     (b) use in, or to build, a product or service that is sold, licensed,
+         offered for a fee or funded by advertising;
+     (c) use to prepare, normalize, train, evaluate or improve data or
+         models (for example speech recognition models) that are used
+         commercially;
+     (d) use in paid consulting, freelance or contract work.
+
+   To get a commercial license, contact Rakhimberdi Rakhmonberdiev at
+   r.raxmonberdiyev10@gmail.com.
+
+3. WHAT YOU MAY NOT DO
+
+   Unless the copyright holder has given you written permission, you may not:
+     (a) modify, translate, adapt or create derivative works of the
+         Software;
+     (b) copy the Software or any part of its source code (including its
+         rules, word lists and regular expressions) into other software,
+         documents or datasets, or re-create it from the source code;
+     (c) sell, rent, sublicense or distribute the Software, except that it
+         may be distributed unmodified, exactly as published by the
+         copyright holder, through the Python Package Index (PyPI) and its
+         mirrors;
+     (d) remove or change this license or any copyright notice.
+
+   Copies made automatically when you install or run the Software as
+   allowed in section 1 or under a commercial license are permitted.
+
+4. TERMINATION
+
+   If you break any of these terms, your permission to use the Software
+   ends automatically, and you must stop using it and delete all copies.
+
+5. NO WARRANTY
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+   OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+   NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR
+   OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+   OTHER DEALINGS IN THE SOFTWARE.
+
+All rights not expressly granted in this license are reserved by the
+copyright holder.

jayron-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+jayron