alifbe 0.2.0__py3-none-any.whl

alifbe/__init__.py

@@ -0,0 +1,85 @@
+"""
+alifbe — robust text normalization utilities for the Uzbek language.
+
+Handles the sharp edges of real-world Uzbek text:
+
+  * Apostrophe chaos (oʻ / gʻ / tutuq belgisi typed with 8+ different glyphs)
+  * The "Isʼhoq" problem (s+h is not always the "sh" digraph)
+  * ş vs ș confusable Unicode lookalikes
+  * The "Turkish I" casing bug
+  * Ambiguous Cyrillic letters (е, ц, ё) that depend on context
+  * The pre-2026 vs. post-reform (Sept 2026 Senate-approved) Latin
+    orthographies -- sh/ch/oʻ/gʻ vs ş/ç/ö/ğ
+  * Cross-script/cross-orthography search-key folding, so "oʻzbek",
+    "özbek", and "ўзбек" are recognized as the same word
+
+Nothing here silently guesses when a guess would corrupt data — ambiguous
+cases are surfaced as warnings so the caller (or a human) can decide.
+"""
+
+from .apostrophe import normalize_apostrophes, TURNED_COMMA, APOSTROPHE
+from .confusables import normalize_confusables, find_confusables, Confusable
+from .case import (
+    uz_lower,
+    uz_upper,
+    uz_title,
+    find_turkish_i_corruption,
+    fix_turkish_i_corruption,
+    CorruptionHit,
+    TURKISH_DOTTED_I,
+    TURKISH_DOTLESS_I,
+)
+from .exceptions_data import SH_BOUNDARY_EXCEPTIONS, is_sh_boundary_word
+from .result import Result, Warning_
+from .transliterate import (
+    TransliterationResult,
+    AmbiguousTransliterationError,
+    latin_to_cyrillic,
+    cyrillic_to_latin,
+)
+from .oldnew import ConversionResult, to_new_latin, to_old_latin
+from .detect import detect_alphabet, AlphabetDetection
+from .searchkey import fold_search_key, fold_search_key_loose
+
+__all__ = [
+    # apostrophe
+    "normalize_apostrophes",
+    "TURNED_COMMA",
+    "APOSTROPHE",
+    # confusables
+    "normalize_confusables",
+    "find_confusables",
+    "Confusable",
+    # case
+    "uz_lower",
+    "uz_upper",
+    "uz_title",
+    "find_turkish_i_corruption",
+    "fix_turkish_i_corruption",
+    "CorruptionHit",
+    "TURKISH_DOTTED_I",
+    "TURKISH_DOTLESS_I",
+    # exceptions
+    "SH_BOUNDARY_EXCEPTIONS",
+    "is_sh_boundary_word",
+    # shared result type
+    "Result",
+    "Warning_",
+    # transliteration (Latin <-> Cyrillic)
+    "TransliterationResult",
+    "AmbiguousTransliterationError",
+    "latin_to_cyrillic",
+    "cyrillic_to_latin",
+    # 2026 reform (old Latin <-> new Latin)
+    "ConversionResult",
+    "to_new_latin",
+    "to_old_latin",
+    # detection
+    "detect_alphabet",
+    "AlphabetDetection",
+    # search keys
+    "fold_search_key",
+    "fold_search_key_loose",
+]
+
+__version__ = "0.2.0"

alifbe/__main__.py

@@ -0,0 +1,95 @@
+"""
+Command-line interface: `python -m alifbe <command> [text]`.
+
+Reads TEXT from the positional argument if given, otherwise from stdin
+(so it works well in pipelines). Warnings, if any, go to stderr so stdout
+stays clean for piping the converted text onward.
+"""
+
+import argparse
+import sys
+
+from . import (
+    normalize_apostrophes,
+    normalize_confusables,
+    uz_upper,
+    uz_lower,
+    uz_title,
+    find_turkish_i_corruption,
+    latin_to_cyrillic,
+    cyrillic_to_latin,
+    to_new_latin,
+    to_old_latin,
+    detect_alphabet,
+    fold_search_key,
+)
+
+
+def _read_text(args) -> str:
+    if args.text is not None:
+        return args.text
+    return sys.stdin.read().rstrip("\n")
+
+
+def _emit(result_text: str, warnings=()) -> None:
+    print(result_text)
+    for w in warnings:
+        print(f"warning: {w.message}", file=sys.stderr)
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="alifbe",
+        description="Normalize and convert Uzbek text (apostrophes, ş/ș confusables, "
+                     "old/new Latin orthography, Cyrillic transliteration).",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    for name in ("normalize-apostrophes", "normalize-confusables",
+                 "upper", "lower", "title",
+                 "to-cyrillic", "to-latin", "to-new-latin", "to-old-latin",
+                 "detect", "fold-key", "check"):
+        p = sub.add_parser(name)
+        p.add_argument("text", nargs="?", help="Text to process (reads stdin if omitted)")
+
+    args = parser.parse_args(argv)
+    text = _read_text(args)
+
+    if args.command == "normalize-apostrophes":
+        _emit(normalize_apostrophes(text))
+    elif args.command == "normalize-confusables":
+        _emit(normalize_confusables(text))
+    elif args.command == "upper":
+        _emit(uz_upper(text))
+    elif args.command == "lower":
+        _emit(uz_lower(text))
+    elif args.command == "title":
+        _emit(uz_title(text))
+    elif args.command == "to-cyrillic":
+        result = latin_to_cyrillic(text)
+        _emit(result.text, result.warnings)
+    elif args.command == "to-latin":
+        result = cyrillic_to_latin(text)
+        _emit(result.text, result.warnings)
+    elif args.command == "to-new-latin":
+        _emit(to_new_latin(text).text)
+    elif args.command == "to-old-latin":
+        _emit(to_old_latin(text).text)
+    elif args.command == "detect":
+        detection = detect_alphabet(text)
+        print(f"{detection.alphabet} (confidence: {detection.confidence})")
+    elif args.command == "fold-key":
+        _emit(fold_search_key(text))
+    elif args.command == "check":
+        hits = find_turkish_i_corruption(text)
+        if hits:
+            for h in hits:
+                print(f"corruption: '{h.char}' ({h.codepoint}) at index {h.index}")
+            return 1
+        print("clean: no Turkish-I corruption found")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

alifbe/apostrophe.py

@@ -0,0 +1,71 @@
+"""
+Normalize the ~8+ different characters people type for Uzbek apostrophes
+into the two *correct* Unicode code points:
+
+  ʻ  U+02BB  MODIFIER LETTER TURNED COMMA   -> used only inside oʻ / gʻ
+  ʼ  U+02BC  MODIFIER LETTER APOSTROPHE     -> the tutuq belgisi (glottal
+                                               stop) used everywhere else,
+                                               e.g. sanʼat, Isʼhoq, mashʼal
+
+People type either of these using: ' (U+0027), ‘ ’ (curly quotes),
+` ´ (grave/acute accent), ʹ (prime), ʻʼ themselves, and more — often
+inconsistently within the same document.
+
+The rule used to decide *which* canonical character a lookalike becomes:
+if it immediately follows o/O/g/G it is part of the oʻ/gʻ letter (turned
+comma); otherwise it is the tutuq belgisi (plain apostrophe).
+"""
+
+import re
+import unicodedata
+
+TURNED_COMMA = "\u02BB"  # ʻ — part of the letters oʻ / gʻ
+APOSTROPHE = "\u02BC"  # ʼ — tutuq belgisi / glottal stop marker
+
+# Every character that, in the wild, gets used to mean "apostrophe" in
+# Uzbek text. This list is intentionally generous.
+_APOSTROPHE_LOOKALIKES = {
+    "'",  # U+0027 APOSTROPHE (ascii)
+    "\u2018",  # ‘ LEFT SINGLE QUOTATION MARK
+    "\u2019",  # ’ RIGHT SINGLE QUOTATION MARK
+    "\u201A",  # ‚ SINGLE LOW-9 QUOTATION MARK
+    "\u201B",  # ‛ SINGLE HIGH-REVERSED-9 QUOTATION MARK
+    "`",  # U+0060 GRAVE ACCENT
+    "\u00B4",  # ´ ACUTE ACCENT
+    "\u02B9",  # ʹ MODIFIER LETTER PRIME
+    "\u02BA",  # ʺ MODIFIER LETTER DOUBLE PRIME
+    "\u02BB",  # ʻ MODIFIER LETTER TURNED COMMA (already canonical form #1)
+    "\u02BC",  # ʼ MODIFIER LETTER APOSTROPHE (already canonical form #2)
+    "\u02BD",  # ʽ MODIFIER LETTER REVERSED COMMA
+    "\u02BE",  # ʾ MODIFIER LETTER RIGHT HALF RING
+    "\u02BF",  # ʿ MODIFIER LETTER LEFT HALF RING
+    "\u02C8",  # ˈ MODIFIER LETTER VERTICAL LINE (stress mark, sometimes misused)
+    "\u02CA",  # ˊ MODIFIER LETTER ACUTE ACCENT
+    "\u02EE",  # ˮ MODIFIER LETTER DOUBLE APOSTROPHE
+    "\u2032",  # ′ PRIME
+    "\u0301",  # combining acute accent (when it ends up standalone)
+}
+
+_PATTERN = re.compile("[" + "".join(re.escape(c) for c in _APOSTROPHE_LOOKALIKES) + "]")
+
+_OG_TRIGGERS = set("oOgG")
+
+
+def normalize_apostrophes(text: str) -> str:
+    """
+    Rewrite every apostrophe-lookalike in `text` to the correct Uzbek
+    character: U+02BB (ʻ) right after o/g, otherwise U+02BC (ʼ).
+
+    NFC-normalizes first so precomposed vs. decomposed forms don't sneak
+    through as false negatives.
+    """
+    text = unicodedata.normalize("NFC", text)
+
+    def repl(match: "re.Match[str]") -> str:
+        idx = match.start()
+        prev_char = text[idx - 1] if idx > 0 else ""
+        if prev_char in _OG_TRIGGERS:
+            return TURNED_COMMA
+        return APOSTROPHE
+
+    return _PATTERN.sub(repl, text)

alifbe/case.py

@@ -0,0 +1,111 @@
+"""
+Locale-independent, explicit-table casing for Uzbek Latin text.
+
+Uzbek Latin uses plain ASCII I/i — it has no dotless ı or dotted İ.
+Those two characters (U+0131 ı, U+0130 İ) belong to Turkish. But a
+surprising number of pipelines silently produce them anyway: ICU / a
+database collation / a JS `.toLocaleUpperCase('tr')` call / an OS with a
+Turkish locale set somewhere upstream all quietly turn plain "i" into
+İ (dotted capital I) or "I" into ı (dotless i). Once that happens, Uzbek
+text is corrupted — "olib" becomes "OLİB" instead of "OLIB", and later
+lookups against the original ASCII string fail.
+
+This module never depends on the platform locale or ICU. It uses fixed,
+explicit translation tables so `uz_upper("olib") == "OLIB"` everywhere,
+always, regardless of what locale the process happens to be running under.
+
+Covers both orthographies in current use:
+  * the pre-2026 Latin alphabet (digraphs sh/ch/ng, letters oʻ/gʻ)
+  * the Latin alphabet approved by Uzbekistan's Senate in Sept 2026
+    (single letters ş/ç/ö/ğ) — not yet in force as of this writing,
+    pending presidential signature
+"""
+
+import re
+from typing import List, NamedTuple
+
+TURKISH_DOTTED_I = "\u0130"  # İ — must never appear in Uzbek text
+TURKISH_DOTLESS_I = "\u0131"  # ı — must never appear in Uzbek text
+
+# Explicit lower -> upper map. Covers the ASCII letters plus every
+# Uzbek-specific character from both orthographies. c/w are included even
+# though they only really appear in loanwords/foreign names — silently
+# leaving them un-cased would be a worse surprise than mapping them.
+_LOWER_TO_UPPER = {
+    "a": "A", "b": "B", "c": "C", "d": "D", "e": "E", "f": "F", "g": "G",
+    "h": "H", "i": "I", "j": "J", "k": "K", "l": "L", "m": "M", "n": "N",
+    "o": "O", "p": "P", "q": "Q", "r": "R", "s": "S", "t": "T", "u": "U",
+    "v": "V", "w": "W", "x": "X", "y": "Y", "z": "Z",
+    "\u02bb": "\u02bb",  # ʻ modifier letter turned comma has no case
+    "\u02bc": "\u02bc",  # ʼ modifier letter apostrophe has no case
+    # 2026 reform letters
+    "\u00f6": "\u00d6",  # ö -> Ö
+    "\u011f": "\u011e",  # ğ -> Ğ
+    "\u015f": "\u015e",  # ş -> Ş
+    "\u00e7": "\u00c7",  # ç -> Ç
+}
+_UPPER_TO_LOWER = {v: k for k, v in _LOWER_TO_UPPER.items()}
+
+_LOWER_TABLE = str.maketrans(_UPPER_TO_LOWER)
+_UPPER_TABLE = str.maketrans(_LOWER_TO_UPPER)
+
+# Digraphs that must be title-cased as a unit ("Sh", not "SH" or "sH")
+# when title-casing a word, e.g. "Shahzoda", "Gʻofur", "Choʻntak".
+_DIGRAPHS = ["sh", "ch", "ng", "o\u02bb", "g\u02bb"]
+_DIGRAPH_RE = re.compile(
+    "|".join(re.escape(d) for d in sorted(_DIGRAPHS, key=len, reverse=True)),
+    re.IGNORECASE,
+)
+
+
+def uz_lower(text: str) -> str:
+    """Lowercase Uzbek Latin text using a fixed table — never locale-dependent."""
+    return text.translate(_LOWER_TABLE)
+
+
+def uz_upper(text: str) -> str:
+    """Uppercase Uzbek Latin text using a fixed table — never locale-dependent."""
+    return text.translate(_UPPER_TABLE)
+
+
+def uz_title(text: str) -> str:
+    """
+    Title-case each word, keeping Uzbek digraphs (sh, ch, ng, oʻ, gʻ) as a
+    single capitalized unit when they start a word, e.g.
+    "shahzoda" -> "Shahzoda", not "SHahzoda" or "SHAHZODA".
+    """
+    def cap_word(word: str) -> str:
+        if not word:
+            return word
+        m = _DIGRAPH_RE.match(word)
+        if m:
+            digraph = m.group(0)
+            rest = word[len(digraph):]
+            return uz_upper(digraph[0]) + uz_lower(digraph[1:]) + uz_lower(rest)
+        return uz_upper(word[0]) + uz_lower(word[1:])
+
+    return re.sub(r"[^\s\-]+", lambda m: cap_word(m.group(0)), text)
+
+
+class CorruptionHit(NamedTuple):
+    index: int
+    char: str
+    codepoint: str
+
+
+def find_turkish_i_corruption(text: str) -> List[CorruptionHit]:
+    """
+    Flag any Turkish dotted/dotless I (İ, ı) in text that claims to be
+    Uzbek — these should never legitimately appear and are a strong signal
+    that a Turkish-locale case operation ran on the text somewhere upstream.
+    """
+    hits = []
+    for i, ch in enumerate(text):
+        if ch in (TURKISH_DOTTED_I, TURKISH_DOTLESS_I):
+            hits.append(CorruptionHit(index=i, char=ch, codepoint=f"U+{ord(ch):04X}"))
+    return hits
+
+
+def fix_turkish_i_corruption(text: str) -> str:
+    """Best-effort repair: İ -> I, ı -> i. Only safe if the source was Uzbek."""
+    return text.replace(TURKISH_DOTTED_I, "I").replace(TURKISH_DOTLESS_I, "i")

alifbe/confusables.py

@@ -0,0 +1,81 @@
+"""
+Detect and normalize Unicode "confusable" characters that *look* identical
+(or nearly identical) but are different code points — meaning two spellings
+of "the same word" silently fail to match in search, sorting, or a database
+unique constraint.
+
+The canonical example: ş (U+015F LATIN SMALL LETTER S WITH CEDILLA, used in
+Turkish/Romanian text people paste in) vs ș (U+0219 LATIN SMALL LETTER S
+WITH COMMA BELOW, the "correct" Romanian character). Uzbek text should not
+contain either — but they leak in from copy-pasted foreign text, and
+because they render almost identically most fonts, humans never notice.
+"""
+
+import unicodedata
+from typing import Dict, List, NamedTuple
+
+# Map of "confusable code point" -> "canonical code point we normalize to".
+# Extend this table as new lookalike pairs are discovered in real data.
+CONFUSABLE_MAP: Dict[str, str] = {
+    "\u0219": "\u015f",  # ș -> ş
+    "\u0218": "\u015e",  # Ș -> Ş
+    "\u021b": "\u0163",  # ț -> ţ
+    "\u021a": "\u0162",  # Ț -> Ţ
+    # Cyrillic/Latin homoglyphs that commonly leak into "Latin" Uzbek text
+    # when copy-pasted from a Cyrillic-typing keyboard layout.
+    "\u0430": "a",  # а (Cyrillic) -> a (Latin)
+    "\u0435": "e",  # е (Cyrillic) -> e (Latin)
+    "\u043e": "o",  # о (Cyrillic) -> o (Latin)
+    "\u0440": "p",  # р (Cyrillic) -> p (Latin)
+    "\u0441": "c",  # с (Cyrillic) -> c (Latin)
+    "\u0445": "x",  # х (Cyrillic) -> x (Latin)
+    "\u0443": "y",  # у (Cyrillic) -> y (Latin)  -- careful: not always desired
+}
+
+
+class Confusable(NamedTuple):
+    index: int
+    char: str
+    codepoint: str
+    suggestion: str
+
+
+def find_confusables(text: str) -> List[Confusable]:
+    """Return every confusable character found in `text`, without changing it."""
+    text = unicodedata.normalize("NFC", text)
+    found = []
+    for i, ch in enumerate(text):
+        if ch in CONFUSABLE_MAP:
+            found.append(
+                Confusable(
+                    index=i,
+                    char=ch,
+                    codepoint=f"U+{ord(ch):04X}",
+                    suggestion=CONFUSABLE_MAP[ch],
+                )
+            )
+    return found
+
+
+def normalize_confusables(text: str, mixed_script_letters: bool = False) -> str:
+    """
+    Rewrite confusable characters to their canonical form.
+
+    mixed_script_letters: if False (default), only fixes diacritic
+    confusables (ş/ș etc.) and leaves bare Cyrillic look-alike letters
+    (а/е/о/р/с/х/у) untouched, since collapsing those can be a real script
+    decision, not just cleanup. Set True to also fold those.
+    """
+    text = unicodedata.normalize("NFC", text)
+    out = []
+    for ch in text:
+        target = CONFUSABLE_MAP.get(ch)
+        if target and (mixed_script_letters or not _is_bare_cyrillic_letter(ch)):
+            out.append(target)
+        else:
+            out.append(ch)
+    return "".join(out)
+
+
+def _is_bare_cyrillic_letter(ch: str) -> bool:
+    return "\u0400" <= ch <= "\u04FF"

alifbe/detect.py

@@ -0,0 +1,66 @@
+"""
+Detect which script/orthography a piece of Uzbek text is written in.
+
+Useful before deciding which conversion function to run, or for flagging
+documents that mix scripts (very common in real-world Uzbek data: a
+Cyrillic paragraph with a Latin brand name, or vice versa).
+"""
+
+import re
+from dataclasses import dataclass
+from typing import Literal
+
+from .apostrophe import TURNED_COMMA
+
+Alphabet = Literal["cyrillic", "old-latin", "new-latin", "mixed", "unknown"]
+
+
+@dataclass(frozen=True)
+class AlphabetDetection:
+    alphabet: Alphabet
+    confidence: float  # 0.0-1.0, rough and heuristic -- not a probability
+
+
+_NEW_LATIN_MARKS = set("öÖğĞşŞçÇ")
+_OLD_LATIN_DIGRAPH_RE = re.compile(r"sh|ch|ng", re.IGNORECASE)
+
+
+def detect_alphabet(text: str) -> AlphabetDetection:
+    """
+    Best-effort guess at which script `text` is written in.
+
+    This is a heuristic over a short string, not a language model — treat
+    the confidence score as "how much signal was there", not as a
+    statistically calibrated probability.
+    """
+    letters = [c for c in text if c.isalpha()]
+    if not letters:
+        return AlphabetDetection("unknown", 0.0)
+
+    cyrillic_count = sum(1 for c in letters if "\u0400" <= c <= "\u04ff")
+    cyrillic_ratio = cyrillic_count / len(letters)
+
+    new_latin_hits = sum(1 for c in text if c in _NEW_LATIN_MARKS)
+    old_latin_hits = len(_OLD_LATIN_DIGRAPH_RE.findall(text)) + text.count(TURNED_COMMA)
+
+    if cyrillic_ratio > 0.5:
+        if new_latin_hits or old_latin_hits:
+            return AlphabetDetection("mixed", round(cyrillic_ratio, 2))
+        return AlphabetDetection("cyrillic", round(cyrillic_ratio, 2))
+
+    if new_latin_hits and old_latin_hits:
+        return AlphabetDetection("mixed", 0.5)
+    if new_latin_hits:
+        return AlphabetDetection("new-latin", min(1.0, 0.6 + 0.1 * new_latin_hits))
+    if old_latin_hits:
+        return AlphabetDetection("old-latin", min(1.0, 0.6 + 0.05 * old_latin_hits))
+    if cyrillic_count:
+        # Some Cyrillic present but not a majority, and no Latin-specific
+        # markers found either -- genuinely mixed.
+        return AlphabetDetection("mixed", round(cyrillic_ratio, 2))
+
+    # Plain ASCII letters only, no digraphs/apostrophes/new letters seen:
+    # could be either orthography (most words look the same in both), or
+    # not Uzbek at all. Low-confidence new-latin is the more useful default
+    # since it's the superset that round-trips cleanly either way.
+    return AlphabetDetection("new-latin", 0.3)

alifbe/exceptions_data.py

@@ -0,0 +1,62 @@
+"""
+Words where "s" and "h" sit next to each other but do NOT form the "sh"
+digraph (ш / ş) — they belong to different morphemes, usually across a
+tutuq belgisi (ʼ) that marks an Arabic-origin glottal stop, e.g.:
+
+    Isʼhoq   (Исҳоқ, the name Isaac/Ishaq)  — is + hoq, not "ish" + "oq"
+    asʼhob   (асҳоб, companions)             — as + hob
+    vasʼhat  (васҳат)                        — vas + hat
+    mashʼal  (машъал, torch)                 — even though "mash" looks
+                                                like a digraph, it's mash + ʼal
+
+The apostrophe usually disambiguates this already — as long as it survives.
+The real danger is upstream code that strips punctuation *before*
+transliterating ("Isʼhoq" -> "Ishoq"), at which point "sh" looks exactly
+like a normal digraph and silently gets merged.
+
+This list lets `is_sh_boundary_word` catch that even when the apostrophe
+is already gone, by checking the word (apostrophes stripped, lowercased)
+against known exceptions. It is necessarily incomplete — a starting point,
+not a linguistic corpus. Pass your own additions via `extra=`.
+"""
+
+from typing import Iterable, Set
+
+# Stored WITHOUT apostrophes, lowercase — the same normalization
+# is_sh_boundary_word applies to its input before comparing.
+SH_BOUNDARY_EXCEPTIONS: Set[str] = {
+    "ishoq",       # Isʼhoq — the name Isaac/Ishaq
+    "ishoqov",     # surname derived from Isʼhoq
+    "ishoqova",
+    "ashob",       # asʼhob — companions (Islamic term)
+    "vashat",      # vasʼhat
+    "mashal",      # mashʼal — torch
+    "mashur",      # mashʼur -- well-known/famous (also seen without apostrophe as "mashhur",
+                    # which genuinely IS a plain digraph word; kept out on purpose, see note below
+}
+
+# NOTE on "mashhur": this is a real ambiguity in the source data itself —
+# "mashhur" (famous) is written with a genuine "sh" digraph in most modern
+# usage and is NOT an s+h boundary word, despite superficially resembling
+# "mashʼal". It's deliberately excluded from the exception set. This kind
+# of collision is exactly why the exception list should be reviewed by a
+# native speaker before being trusted for anything production-critical.
+
+
+def is_sh_boundary_word(word: str, extra: Iterable[str] = ()) -> bool:
+    """
+    Check whether `word` (with or without an apostrophe already present)
+    is a known case where s+h is a morpheme boundary, not the "sh"/"ş"
+    digraph.
+
+    Matching is done on the lowercased word with apostrophes stripped, so
+    it catches the word whether or not the apostrophe survived upstream.
+    """
+    normalized = (
+        word.lower()
+        .replace("\u02bc", "")  # ʼ
+        .replace("\u02bb", "")  # ʻ
+        .replace("'", "")
+    )
+    exception_set = SH_BOUNDARY_EXCEPTIONS | {e.lower() for e in extra}
+    return normalized in exception_set

alifbe/oldnew.py

@@ -0,0 +1,110 @@
+"""
+Convert between the pre-2026 Uzbek Latin alphabet (26 letters + 3 digraphs
+sh/ch/ng + apostrophe) and the alphabet approved by Uzbekistan's Senate on
+10 September 2026 (28 letters + 1 apostrophe):
+
+    Old      New
+    ---      ---
+    sh, Sh   ş, Ş
+    ch, Ch   ç, Ç
+    oʻ, Oʻ   ö, Ö
+    gʻ, Gʻ   ğ, Ğ
+
+"ng" is dropped as a distinct letter but is NOT rewritten — it was already
+spelled with plain n+g and continues to be; only how the sound is taught/
+classified changes, not the spelling. The tutuq belgisi (ʼ) is unaffected;
+it remains the alphabet's one apostrophe under the new system too.
+
+STATUS: as of this writing the amending law has been approved by the
+Senate and sent to the president, but is **not yet in force**. Treat
+`to_new_latin` as a forward-looking convenience, not a claim about which
+spelling is currently official — check the latest guidance from lex.uz or
+the Uzbek Ministry of Education before assuming this is mandatory.
+"""
+
+import re
+from typing import Iterable, List, Tuple
+
+from .apostrophe import normalize_apostrophes, TURNED_COMMA
+from .result import Result
+
+ConversionResult = Result
+
+# Matches http(s) URLs, emails, and `backtick-quoted` spans -- things that
+# are very unlikely to be running Uzbek prose and where an automatic sh/ch
+# rewrite is more likely to break a slug/identifier than help.
+_AUTO_PROTECT_PATTERN = re.compile(
+    r"https?://\S+"
+    r"|\b[\w.+-]+@[\w-]+\.[\w.-]+\b"
+    r"|`[^`]*`"
+)
+
+_PLACEHOLDER_TEMPLATE = "\uE000{index}\uE001"  # private-use-area markers
+_PLACEHOLDER_RE = re.compile("\uE000(\\d+)\uE001")
+
+
+def _protect(text: str, protected_terms: Iterable[str], protect_spans: bool
+             ) -> Tuple[str, List[str]]:
+    """Replace protected substrings with placeholders; return (text, originals)."""
+    originals: List[str] = []
+
+    def stash(match_text: str) -> str:
+        originals.append(match_text)
+        return _PLACEHOLDER_TEMPLATE.format(index=len(originals) - 1)
+
+    # Protect explicit terms first (longest first, so overlapping/prefix
+    # terms don't partially shadow each other), then automatic spans.
+    for term in sorted({t for t in protected_terms if t}, key=len, reverse=True):
+        if term in text:
+            text = text.replace(term, stash(term))
+    if protect_spans:
+        text = _AUTO_PROTECT_PATTERN.sub(lambda m: stash(m.group(0)), text)
+    return text, originals
+
+
+def _restore(text: str, originals: List[str]) -> str:
+    return _PLACEHOLDER_RE.sub(lambda m: originals[int(m.group(1))], text)
+
+
+def to_new_latin(text: str, protect_spans: bool = True,
+                  protected_terms: Iterable[str] = ()) -> Result:
+    """
+    Convert pre-2026 Uzbek Latin text to the 2026-reform Latin alphabet.
+
+    protect_spans:    if True (default), URLs, emails, and `backtick`
+                       spans are left untouched.
+    protected_terms:  exact substrings (e.g. brand names) to leave
+                       untouched, e.g. protected_terms=["MyShop"] keeps
+                       "MyShop" from becoming "MyŞop".
+    """
+    # Protect spans FIRST, using the raw input -- apostrophe normalization
+    # would otherwise eat backticks (U+0060 is itself an apostrophe
+    # lookalike) before the protection regex ever gets to see them.
+    protected, originals = _protect(text, protected_terms, protect_spans)
+    protected = normalize_apostrophes(protected)
+
+    converted = re.sub(r"sh", lambda m: _case(m.group(0), "ş", "Ş"), protected, flags=re.IGNORECASE)
+    converted = re.sub(r"ch", lambda m: _case(m.group(0), "ç", "Ç"), converted, flags=re.IGNORECASE)
+    converted = re.sub(f"[oO]{TURNED_COMMA}", lambda m: "Ö" if m.group(0)[0].isupper() else "ö", converted)
+    converted = re.sub(f"[gG]{TURNED_COMMA}", lambda m: "Ğ" if m.group(0)[0].isupper() else "ğ", converted)
+
+    result_text = _restore(converted, originals)
+    return Result(text=result_text, warnings=())
+
+
+def to_old_latin(text: str, protect_spans: bool = True,
+                  protected_terms: Iterable[str] = ()) -> Result:
+    """Convert 2026-reform Uzbek Latin text back to the pre-2026 alphabet."""
+    protected, originals = _protect(text, protected_terms, protect_spans)
+
+    converted = protected.replace("ş", "sh").replace("Ş", "Sh")
+    converted = converted.replace("ç", "ch").replace("Ç", "Ch")
+    converted = converted.replace("ö", f"o{TURNED_COMMA}").replace("Ö", f"O{TURNED_COMMA}")
+    converted = converted.replace("ğ", f"g{TURNED_COMMA}").replace("Ğ", f"G{TURNED_COMMA}")
+
+    result_text = _restore(converted, originals)
+    return Result(text=result_text, warnings=())
+
+
+def _case(matched: str, lower: str, upper: str) -> str:
+    return upper if any(c.isupper() for c in matched) else lower

alifbe/result.py

@@ -0,0 +1,42 @@
+"""
+Shared result types.
+
+Every conversion function in alifbe returns the same shape: the converted
+text, plus a tuple of warnings for anything it wasn't fully certain about.
+Both are frozen (immutable) — a Result you get back can't be silently
+mutated somewhere downstream and cause confusing bugs.
+"""
+
+from dataclasses import dataclass, field
+from typing import Tuple
+
+
+@dataclass(frozen=True)
+class Warning_:
+    """
+    One thing the conversion wasn't fully sure about.
+
+    index:   code-point offset into the *input* text where this applies.
+    length:  how many input code points this warning covers (usually 1).
+    rule:    a short machine-readable identifier, e.g. "cyrillic.e.positional",
+             so callers can filter/handle specific kinds of ambiguity.
+    message: a human-readable explanation.
+    """
+    index: int
+    length: int
+    rule: str
+    message: str
+
+
+@dataclass(frozen=True)
+class Result:
+    """The output of a conversion: the text, and zero or more warnings."""
+    text: str
+    warnings: Tuple[Warning_, ...] = field(default_factory=tuple)
+
+    def __str__(self) -> str:
+        return self.text
+
+    def __bool__(self) -> bool:
+        # A Result is "clean" (truthy) when there were no warnings.
+        return not self.warnings

alifbe/searchkey.py

@@ -0,0 +1,61 @@
+"""
+Canonical search keys for Uzbek text.
+
+The whole point of this library is that "the same word" can arrive typed
+eight different ways. That's a display problem for humans, but it's a
+*data* problem for search indexes, database unique constraints, and
+deduplication: "oʻzbek", "o'zbek", "özbek", and "ўзбек" all mean the same
+thing, but a naive `.lower()` treats them as four different strings.
+
+fold_search_key() produces one canonical Latin (new-orthography), NFC,
+locale-safe-lowercased string regardless of which script or apostrophe
+style the input used, so equality/hashing/indexing "just works".
+"""
+
+import unicodedata
+
+from .apostrophe import normalize_apostrophes
+from .case import uz_lower
+from .confusables import normalize_confusables
+from .detect import detect_alphabet
+from .oldnew import to_new_latin
+from .transliterate import cyrillic_to_latin
+
+
+def fold_search_key(text: str) -> str:
+    """
+    Canonical search key: same output for the same word regardless of
+    script (Cyrillic/Latin) or apostrophe style. Lossless with respect to
+    the letters ö/ğ/ş/ç/oʻ/gʻ/etc. — different *sounding* words won't
+    collide, only different spellings of the same word do.
+    """
+    text = normalize_apostrophes(text)
+    text = normalize_confusables(text)
+
+    detection = detect_alphabet(text)
+    if detection.alphabet == "cyrillic":
+        text = cyrillic_to_latin(text, on_ambiguous="best_guess", new_orthography=True).text
+    else:
+        # old-latin, new-latin, mixed, or unknown: run the old->new pass
+        # regardless. It's idempotent on text that's already new-latin,
+        # and it upgrades any old-latin digraphs it finds in mixed text.
+        text = to_new_latin(text, protect_spans=False).text
+
+    return uz_lower(text)
+
+
+def fold_search_key_loose(text: str) -> str:
+    """
+    Like fold_search_key(), but also strips diacritics (ö->o, ğ->g, ş->s,
+    ç->c, apostrophes removed entirely).
+
+    This is intentionally lossy: "sen" (you) and "şen" (cheerful) would
+    fold to the same loose key. Only use this for "did you maybe mean"
+    style fuzzy search, never as a unique key or for anything where a
+    false match would be a real problem.
+    """
+    key = fold_search_key(text)
+    decomposed = unicodedata.normalize("NFKD", key)
+    stripped = "".join(c for c in decomposed if not unicodedata.combining(c))
+    stripped = stripped.replace("\u02bb", "").replace("\u02bc", "").replace("'", "")
+    return unicodedata.normalize("NFC", stripped)

alifbe/transliterate.py

@@ -0,0 +1,306 @@
+"""
+Uzbek Latin <-> Cyrillic transliteration.
+
+Two hard problems this module refuses to paper over:
+
+1. The "Isʼhoq" problem: "sh" is usually the digraph for ш, but not when
+   s and h belong to different morphemes. We check the apostrophe *and*
+   a known-word exception list before ever treating "sh" as one letter.
+
+2. Cyrillic е, ц, ё are position/word-dependent and sometimes genuinely
+   ambiguous (native vs. borrowed word spelling rules differ). Rather
+   than guess silently, cyrillic_to_latin() always returns a `warnings`
+   tuple alongside the text, and can be configured to raise instead.
+
+Digraph matching is done with case-insensitive regexes rather than an
+explicit table of pre-listed case combinations (`"Sh"`, `"SH"`, ...) —
+a fixed table is exactly the kind of thing that quietly mishandles an
+unusual input like `"sH"` or text typed with caps-lock stuck on. The
+regex approach handles every case combination by construction.
+"""
+
+import re
+import unicodedata
+from typing import Iterable, List, Tuple
+
+from .apostrophe import normalize_apostrophes, TURNED_COMMA, APOSTROPHE
+from .exceptions_data import is_sh_boundary_word
+from .result import Result, Warning_
+
+
+class AmbiguousTransliterationError(ValueError):
+    """Raised when on_ambiguous='raise' and an ambiguous letter is hit."""
+
+
+# Backwards/forwards-compatible alias: transliteration results and
+# old<->new-latin conversion results share the same shape.
+TransliterationResult = Result
+
+
+# ---------------------------------------------------------------------------
+# Latin -> Cyrillic
+# ---------------------------------------------------------------------------
+
+_L2C_SINGLES = {
+    "a": "а", "b": "б", "c": "к", "d": "д", "e": "е", "f": "ф", "g": "г",
+    "h": "ҳ", "i": "и", "j": "ж", "k": "к", "l": "л", "m": "м", "n": "н",
+    "o": "о", "p": "п", "q": "қ", "r": "р", "s": "с", "t": "т", "u": "у",
+    "v": "в", "w": "в", "x": "х", "y": "й", "z": "з",
+    APOSTROPHE: "ъ",
+}
+
+
+def _cased(latin_lower: str, cyr_lower: str, cyr_upper: str):
+    """Build a (compiled pattern, lower, upper) triple for a digraph."""
+    return re.compile(re.escape(latin_lower), re.IGNORECASE), cyr_lower, cyr_upper
+
+
+_L2C_DIGRAPH_SPECS = [
+    _cased("sh", "ш", "Ш"),
+    _cased("ch", "ч", "Ч"),
+    _cased("ng", "нг", "Нг"),
+    _cased("yo", "ё", "Ё"),
+    _cased("yu", "ю", "Ю"),
+    _cased("ya", "я", "Я"),
+]
+
+# oʻ / gʻ: the turned comma itself carries no case, so casing is decided
+# by the o/g alone.
+_OG_PATTERN = re.compile(f"([oOgG]){TURNED_COMMA}")
+_OG_TARGET = {"o": "ў", "g": "ғ"}
+
+
+def _apply_case(match_text: str, lower: str, upper: str) -> str:
+    # Any uppercase letter in the matched span means "this is a capital
+    # letter" -- there's no meaningful notion of "SH" vs "Sh" once it
+    # becomes a single Cyrillic/new-Latin character, so any-uppercase wins.
+    return upper if any(c.isupper() for c in match_text) else lower
+
+
+_WORD_RE = re.compile(r"[^\W\d_]+", re.UNICODE)
+
+
+def latin_to_cyrillic(text: str, extra_sh_exceptions: Iterable[str] = ()) -> Result:
+    """
+    Convert Uzbek Latin text (old or new orthography) to Cyrillic.
+
+    Correctly refuses to merge "sh" into ш when the word is a known
+    s+h morpheme-boundary exception (e.g. "Isʼhoq") or when an
+    apostrophe already separates them, even if a prior "cleanup" step
+    stripped the apostrophe from an otherwise-recognized word.
+
+    Also understands the 2026-reform single letters (ş/ç/ö/ğ) directly,
+    so text doesn't need to be downgraded to the old orthography first.
+    """
+    text = normalize_apostrophes(text)
+    warnings: List[Warning_] = []
+    out_parts: List[str] = []
+
+    last = 0
+    for m in _WORD_RE.finditer(text):
+        out_parts.append(text[last:m.start()])
+        word = m.group(0)
+        out_parts.append(_convert_word_l2c(word, warnings, m.start(), extra_sh_exceptions))
+        last = m.end()
+    out_parts.append(text[last:])
+
+    return Result(text="".join(out_parts), warnings=tuple(warnings))
+
+
+def _convert_word_l2c(word: str, warnings: List[Warning_], base_index: int,
+                       extra_sh_exceptions: Iterable[str]) -> str:
+    protect_sh = is_sh_boundary_word(word, extra=extra_sh_exceptions)
+    if protect_sh and APOSTROPHE not in word and re.search(r"sh", word, re.IGNORECASE):
+        warnings.append(Warning_(
+            index=base_index,
+            length=len(word),
+            rule="latin.sh_boundary.no_apostrophe",
+            message=(
+                f"'{word}' looks like it contains the 'sh' digraph but is a known "
+                f"s+h morpheme-boundary word with no apostrophe present. Converted "
+                f"as с+ҳ, not ш. Consider restoring the ʼ (tutuq belgisi) at the source."
+            ),
+        ))
+
+    i = 0
+    n = len(word)
+    result: List[str] = []
+    while i < n:
+        if protect_sh and word[i:i + 2].lower() == "sh":
+            result.append(_L2C_SINGLES.get(word[i].lower(), word[i]))
+            result.append(_L2C_SINGLES.get(word[i + 1].lower(), word[i + 1]))
+            i += 2
+            continue
+
+        # New-orthography single letters (ş ç ö ğ)
+        new_letter = _NEW_LATIN_TO_CYR.get(word[i].lower())
+        if new_letter is not None:
+            result.append(new_letter.upper() if word[i].isupper() else new_letter)
+            i += 1
+            continue
+
+        og = _OG_PATTERN.match(word, i)
+        if og:
+            base = og.group(1).lower()
+            cyr = _OG_TARGET[base]
+            result.append(cyr.upper() if og.group(1).isupper() else cyr)
+            i = og.end()
+            continue
+
+        matched_digraph = False
+        for pattern, lower_cyr, upper_cyr in _L2C_DIGRAPH_SPECS:
+            m = pattern.match(word, i)
+            if m:
+                result.append(_apply_case(m.group(0), lower_cyr, upper_cyr))
+                i = m.end()
+                matched_digraph = True
+                break
+        if matched_digraph:
+            continue
+
+        ch = word[i]
+        mapped = _L2C_SINGLES.get(ch.lower())
+        if mapped is not None:
+            result.append(mapped.upper() if ch.isupper() else mapped)
+        else:
+            result.append(ch)
+        i += 1
+
+    return "".join(result)
+
+
+# New-orthography (post-reform) single letters, lowercase-keyed.
+_NEW_LATIN_TO_CYR = {
+    "ş": "ш",
+    "ç": "ч",
+    "ö": "ў",
+    "ğ": "ғ",
+}
+
+
+# ---------------------------------------------------------------------------
+# Cyrillic -> Latin  (with explicit ambiguity warnings for е, ц, ё)
+# ---------------------------------------------------------------------------
+
+_C2L_SIMPLE = {
+    "а": "a", "б": "b", "в": "v", "г": "g", "д": "d", "ж": "j",
+    "з": "z", "и": "i", "й": "y", "к": "k", "л": "l", "м": "m",
+    "н": "n", "о": "o", "п": "p", "р": "r", "с": "s", "т": "t",
+    "у": "u", "ф": "f", "х": "x", "ш": "sh", "ч": "ch", "қ": "q",
+    "ғ": f"g{TURNED_COMMA}", "ў": f"o{TURNED_COMMA}", "ҳ": "h",
+    "ъ": APOSTROPHE, "ь": "", "ю": "yu", "я": "ya",
+}
+
+# Letters whose correct romanization genuinely depends on position or on
+# whether the word is native vs. borrowed. We never guess these silently
+# without recording why.
+_AMBIGUOUS = {"е", "ц", "ё"}
+
+_VOWELS = set("аеёиоуўэюя")
+
+
+def cyrillic_to_latin(text: str, on_ambiguous: str = "warn",
+                       new_orthography: bool = False) -> Result:
+    """
+    Convert Uzbek Cyrillic text to Latin.
+
+    on_ambiguous:
+      "warn"       (default) - use a reasonable positional best-guess for
+                    е (ye at word start / after a vowel or ъ/ь, else e),
+                    and for ц/ё fall back to a documented default — but
+                    always record a Warning_ so the caller knows a guess
+                    was made.
+      "raise"      - raise AmbiguousTransliterationError on the first
+                     ambiguous letter instead of guessing.
+      "best_guess" - same positional guessing as "warn", but does not
+                     populate the warnings tuple.
+
+    new_orthography: if True, emit the 2026-reform single letters
+      (ş/ç/ö/ğ) instead of the digraphs (sh/ch/oʻ/gʻ).
+    """
+    if on_ambiguous not in ("warn", "raise", "best_guess"):
+        raise ValueError("on_ambiguous must be 'warn', 'raise', or 'best_guess'")
+
+    text = unicodedata.normalize("NFC", text)
+    warnings: List[Warning_] = []
+    out: List[str] = []
+
+    for i, ch in enumerate(text):
+        lower = ch.lower()
+        is_upper = ch.isupper()
+
+        if lower in _AMBIGUOUS:
+            if on_ambiguous == "raise":
+                raise AmbiguousTransliterationError(
+                    f"Ambiguous Cyrillic letter '{ch}' at index {i} "
+                    f"(context: ...{text[max(0, i - 5):i + 6]}...). "
+                    f"Its correct Latin form depends on word origin/position; "
+                    f"resolve manually or pass on_ambiguous='best_guess'."
+                )
+            guess, rule, note = _guess_ambiguous(lower, text, i)
+            out.append(guess.upper() if is_upper else guess)
+            if on_ambiguous == "warn":
+                warnings.append(Warning_(index=i, length=1, rule=rule, message=note))
+            continue
+
+        mapped = _C2L_SIMPLE.get(lower)
+        if mapped is None:
+            out.append(ch)
+            continue
+
+        if is_upper and mapped:
+            mapped = mapped[0].upper() + mapped[1:]
+        out.append(mapped)
+
+    result_text = "".join(out)
+    if new_orthography:
+        result_text = _digraphs_to_new_letters(result_text)
+
+    return Result(text=result_text, warnings=tuple(warnings))
+
+
+def _digraphs_to_new_letters(text: str) -> str:
+    text = re.sub(r"sh", lambda m: _apply_case(m.group(0), "ş", "Ş"), text, flags=re.IGNORECASE)
+    text = re.sub(r"ch", lambda m: _apply_case(m.group(0), "ç", "Ç"), text, flags=re.IGNORECASE)
+    text = re.sub(f"[oO]{TURNED_COMMA}", lambda m: "Ö" if m.group(0)[0].isupper() else "ö", text)
+    text = re.sub(f"[gG]{TURNED_COMMA}", lambda m: "Ğ" if m.group(0)[0].isupper() else "ğ", text)
+    return text
+
+
+def _guess_ambiguous(lower_ch: str, text: str, i: int) -> Tuple[str, str, str]:
+    prev = text[i - 1] if i > 0 else ""
+    is_word_start = i == 0 or not prev.isalpha()
+
+    if lower_ch == "е":
+        ye_context = is_word_start or prev.lower() in _VOWELS or prev.lower() in ("ъ", "ь")
+        latin = "ye" if ye_context else "e"
+        rule = "cyrillic.e.positional"
+        note = (
+            f"'е' romanized as '{latin}' based on position "
+            f"({'word-initial/after vowel or ъ-ь' if ye_context else 'after consonant'}); "
+            f"this is a positional heuristic, not a certainty for borrowed words."
+        )
+        return latin, rule, note
+
+    if lower_ch == "ц":
+        # No reliable positional rule: some words render as 's', direct
+        # borrowings (proper nouns, scientific/technical terms) as 'ts'.
+        # We default to 'ts' (the more literal, information-preserving
+        # choice) and flag it loudly.
+        rule = "cyrillic.ts.ambiguous"
+        note = (
+            "'ц' has no reliable rule (can be 's' or 'ts' depending on the "
+            "word's origin). Defaulted to 'ts' — verify against a dictionary "
+            "for native-feeling words."
+        )
+        return "ts", rule, note
+
+    if lower_ch == "ё":
+        rule = "cyrillic.yo.foreign"
+        note = (
+            "'ё' is not part of the native Uzbek Cyrillic alphabet and "
+            "usually only appears in Russian loanwords; defaulted to 'yo'."
+        )
+        return "yo", rule, note
+
+    return lower_ch, "", ""

alifbe-0.2.0.dist-info/METADATA

@@ -0,0 +1,177 @@
+Metadata-Version: 2.4
+Name: alifbe
+Version: 0.2.0
+Summary: Robust normalization for real-world Uzbek text: apostrophe chaos, sh/s+h word boundaries, ş/ș confusables, the Turkish-I casing bug, ambiguous Cyrillic letters, and conversion between the pre-2026 and Sept-2026-reform Latin alphabets.
+Author: you
+License: MIT
+Project-URL: Homepage, https://github.com/yourname/alifbe
+Project-URL: Repository, https://github.com/yourname/alifbe
+Keywords: uzbek,unicode,normalization,transliteration,i18n,nlp,cyrillic,latin-alphabet
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Topic :: Software Development :: Internationalization
+Classifier: Typing :: Typed
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# alifbe
+
+Robust text normalization for real-world Uzbek text — the eight problems
+that quietly corrupt Uzbek data in most pipelines.
+
+| Problem | What alifbe does |
+|---|---|
+| 8+ different "apostrophe" characters (`'` `‘` `’` `` ` `` `´` ...) all meaning oʻ / gʻ / tutuq belgisi | Normalizes all of them to the two *correct* Unicode letters, based on context |
+| `"Isʼhoq"` → `"Işoq"` | Detects s+h morpheme boundaries (via apostrophe *and* a known-word list) so `sh` isn't wrongly merged into one letter |
+| `ş` vs `ș` (cedilla vs. comma-below — different code points, identical glyph) | Detects and normalizes confusable characters so search/dedup actually works |
+| The "Turkish I" bug (`.upper()`/`.lower()` under some locales turns `i` into `İ`) | Locale-independent, explicit-table casing that never produces `İ`/`ı` |
+| Cyrillic `е`, `ц`, `ё` are position/origin-dependent | Transliteration always returns warnings for ambiguous letters instead of silently guessing (and can `raise` instead, if you'd rather fail loudly) |
+| The **Sept 2026 alphabet reform** (sh→ş, ch→ç, oʻ→ö, gʻ→ğ) | `to_new_latin()` / `to_old_latin()` convert between the two orthographies, with brand-name/URL/code protection |
+| "Which script is this text even in?" | `detect_alphabet()` — cyrillic / old-latin / new-latin / mixed / unknown |
+| "oʻzbek", "özbek", and "ўзбек" are the same word, but `==` doesn't think so | `fold_search_key()` gives every spelling the same canonical key |
+
+## Install
+
+Not yet published to PyPI — the name `alifbe` is free, checked via
+`pip download alifbe` returning no match, but publishing itself is a
+manual step (PyPI account + 2FA + `twine upload`). Install locally for now:
+
+```bash
+pip install /path/to/alifbe          # normal install
+pip install -e /path/to/alifbe       # editable, for development
+```
+
+This also registers an `alifbe` command-line tool (see below).
+
+## Quick tour
+
+```python
+import alifbe as uz
+
+# 1. Apostrophes: every variant collapses to the correct, same string
+uz.normalize_apostrophes("o'zbek")     # -> "oʻzbek"  (U+02BB, turned comma)
+uz.normalize_apostrophes("o‘zbek")     # -> "oʻzbek"  (same result)
+uz.normalize_apostrophes("san'at")     # -> "sanʼat"  (U+02BC — different rule, not after o/g)
+
+# 2. "Isʼhoq" stays "Isʼhoq" -- sh is not wrongly merged into ш
+uz.latin_to_cyrillic("Isʼhoq").text        # -> "Исъҳоқ"  (not "Ишоқ")
+uz.latin_to_cyrillic("Ishoq").warnings     # non-empty: flags this as a known
+                                            # s+h boundary word even without
+                                            # the apostrophe
+
+# 3. ş vs ș
+uz.find_confusables("Kraiova munșasi")           # -> [Confusable(char='ș', codepoint='U+0219', ...)]
+uz.normalize_confusables("munşa") == uz.normalize_confusables("munșa")  # -> True
+
+# 4. Turkish I bug
+uz.uz_upper("olib")                        # -> "OLIB"  (never "OLİB")
+uz.find_turkish_i_corruption("OLİB")       # -> flags the İ as corruption
+uz.fix_turkish_i_corruption("OLİB")        # -> "OLIB"
+
+# 5. Ambiguous Cyrillic letters warn instead of silently guessing
+result = uz.cyrillic_to_latin("центр")
+result.text                                # -> "tsentr"  (best guess)
+result.warnings                            # -> non-empty, explains ц is ambiguous
+uz.cyrillic_to_latin("ёлғон", on_ambiguous="raise")  # -> raises instead
+
+# 6. The Sept 2026 alphabet reform
+uz.to_new_latin("Shahzoda Oʻzbekistonda choy ichdi").text
+# -> "Şahzoda Özbekistonda çoy içdi"
+uz.to_old_latin("Şahzoda Özbekistonda çoy içdi").text
+# -> "Shahzoda Oʻzbekistonda choy ichdi"
+
+# ... with brand names / URLs / code protected from conversion
+uz.to_new_latin("MyShop: sotib oling", protected_terms=["MyShop"]).text
+# -> "MyShop: sotib oling"   (MyShop untouched, rest still converts if applicable)
+uz.to_new_latin("See https://x.com/shahar for info").text
+# -> "See https://x.com/shahar for info"   (URL untouched)
+
+# 7. What script is this?
+uz.detect_alphabet("Shahzoda")   # -> AlphabetDetection(alphabet='old-latin', confidence=0.65)
+uz.detect_alphabet("Şahzoda")    # -> AlphabetDetection(alphabet='new-latin', confidence=0.7)
+uz.detect_alphabet("Шаҳзода")    # -> AlphabetDetection(alphabet='cyrillic', confidence=1.0)
+
+# 8. One search key regardless of script or apostrophe style
+uz.fold_search_key("o'zbek") == uz.fold_search_key("özbek") == uz.fold_search_key("ўзбек")
+# -> True (all fold to "özbek")
+```
+
+## Command line
+
+```bash
+echo "o'zbek" | alifbe normalize-apostrophes          # oʻzbek
+alifbe to-new-latin "Shahzoda choy ichdi"              # Şahzoda çoy içdi
+alifbe to-cyrillic "Ishoq"                              # Исъҳоқ  (+ warning on stderr)
+alifbe detect "Шаҳзода"                                 # cyrillic (confidence: 1.0)
+alifbe fold-key "oʻzbek"                                 # özbek
+alifbe check "OLİB"                                       # corruption: 'İ' (U+0130) at index 2
+```
+
+Every subcommand reads from the positional argument if given, or stdin
+otherwise — so it pipes cleanly. Conversion warnings go to stderr, so
+stdout stays clean for piping the result onward.
+
+## Design principle
+
+Every function that could plausibly get something wrong either:
+- makes the correct choice deterministically (apostrophes, casing,
+  alphabet-reform digraphs), or
+- tells you it's not sure, instead of guessing silently (Cyrillic
+  е/ц/ё, the sh/s+h boundary when no apostrophe survives).
+
+That second category is deliberate: a library that *always* looks
+confident is more dangerous than one that sometimes says "I'm not sure,
+here's my best guess and why." All conversion functions return the same
+`Result(text, warnings)` shape — `warnings` is empty when the library is
+confident, and populated (with a machine-readable `rule` plus a
+human-readable `message`) when it made a judgment call.
+
+## On the Sept 2026 alphabet reform
+
+Uzbekistan's Senate approved a bill on 10 September 2026 replacing the
+sh/ch/oʻ/gʻ digraphs with single letters ş/ç/ö/ğ. As of this writing the
+bill has been sent to the president and is **not yet in force** — school
+materials are expected to transition starting 2027. `to_new_latin()` is a
+forward-looking convenience, not a claim about which spelling is
+currently mandatory. Check lex.uz or the Ministry of Education for the
+authoritative status before treating conversion as required.
+
+## Known simplifications (read before relying on this for anything critical)
+
+- `SH_BOUNDARY_EXCEPTIONS` (in `exceptions_data.py`) is a small starting
+  list, not a linguistic corpus — extend it via `extra=` for your data,
+  and get a native speaker to review it before production use.
+- Cyrillic `ц` has no reliable positional rule (can be "s" or "ts"
+  depending on the word's origin); alifbe defaults to "ts" and always
+  warns. Cyrillic `е` uses a positional heuristic (ye at word start/after
+  a vowel, else e) which is usually right but isn't a certainty for
+  borrowed words.
+- `latin_to_cyrillic("c")` maps to "к" (since old-orthography Uzbek Latin
+  has no standalone "c" outside the "ch" digraph) — for text with
+  loanwords spelled with a bare "c", double-check the output.
+- `fold_search_key_loose()` is intentionally lossy (strips ö/ğ/ş/ç
+  diacritics and apostrophes) — never use it as a unique key, only for
+  fuzzy "did you mean" style matching.
+
+## Extending the sh-boundary word list
+
+```python
+uz.latin_to_cyrillic("Asʼhad", extra_sh_exceptions={"ashad"})
+```
+
+## Tests
+
+```bash
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+102 tests across normalization, transliteration, the alphabet-reform
+converter, script detection, search-key folding, and the CLI.

alifbe-0.2.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+alifbe/__init__.py,sha256=dTQxZxFW_zZyUtMKsbqSKfVlzNTRIPD5UOJA_SiFsDc,2516
+alifbe/__main__.py,sha256=rS5kqQ4tMWCxHnwzompzTWfTrQDMNUnmNYXREV4yHmY,2963
+alifbe/apostrophe.py,sha256=cFJBs6gBdQ5SDaPwKZMuEJZask04tdqNw4wooFuWsBI,2861
+alifbe/case.py,sha256=yj0yLNlfGAOGJlyJyZWWet5LzJwfCBa9unLuZwSjRfE,4468
+alifbe/confusables.py,sha256=DePw1dgS-4EMhdVjsRZg74P-1sHgL6NUbXn9WILebFM,2984
+alifbe/detect.py,sha256=kL7O1_hsnM_qmEih7RfsCoU2Wz0o-gwnqNIGNh9YZc0,2516
+alifbe/exceptions_data.py,sha256=JnhaHqLSMSMTVHpEOya4p1L1y0jM2bOM3zYcGfYVgPs,2893
+alifbe/oldnew.py,sha256=NufVW5VEgUVl0FKZdL27dFg9pDXMS413i_qvtNz9Hnw,4737
+alifbe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+alifbe/result.py,sha256=-nBOvmBZeR6mUIbJ8Itbxt3WvW3qulkFLPEDCCEwoN4,1273
+alifbe/searchkey.py,sha256=ofICm8YgRKiIdLwry-Gc0XZicVy77YnD1fm0tf3Mg3o,2510
+alifbe/transliterate.py,sha256=OH-fzfxtuy6gIqs9HgwWLXRVDKpeajiNz70kbLUgizo,11708
+alifbe-0.2.0.dist-info/licenses/LICENSE,sha256=ESYyLizI0WWtxMeS7rGVcX3ivMezm-HOd5WdeOh-9oU,1056
+alifbe-0.2.0.dist-info/METADATA,sha256=l8YZ_ZdWbgo1gpRbWqAfkW_MXQsFw0tPAb_8YcTOD4w,8999
+alifbe-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+alifbe-0.2.0.dist-info/entry_points.txt,sha256=2pwRxqXw7Tr06nX21Z-7gnkUJfvt9Lv-wYUHB7qNPrE,48
+alifbe-0.2.0.dist-info/top_level.txt,sha256=hmzgmWiQBnPNViz9FN9RGr2uZDgKt5GWaWLiBqkndkw,7
+alifbe-0.2.0.dist-info/RECORD,,

alifbe-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

alifbe-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+alifbe = alifbe.__main__:main

alifbe-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

alifbe-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+alifbe