hindi-normalize 0.1.0__py3-none-any.whl

hindi_normalize/__init__.py

@@ -0,0 +1,56 @@
+"""
+hindi_normalize — convert Hindi number words to digits and normalize Devanagari
+ASR/OCR text. Zero dependencies beyond the Python standard library.
+
+Quick start::
+
+    from hindi_normalize import normalize_transcript, convert_numbers
+    from hindi_normalize import normalize_devanagari, replace_terms
+
+    normalize_transcript("आपका BP एक सौ दस बटा सत्तर है")  # 'आपका BP 110/70 है'
+    convert_numbers("नौ सौ निन्यानवे")                      # '999'
+    normalize_devanagari("वजन ५८ किलो | ठीक है")            # 'वजन 58 किलो। ठीक है'
+    replace_terms("बीबी ठीक है")                            # 'BP ठीक है'
+
+Building blocks (all importable from the top level):
+
+  * numbers      — parse_hindi_number, convert_numbers, WORD_TO_NUM
+  * devanagari   — normalize_devanagari, strip_zero_width, digit conversion,
+                   normalize_danda, fold_nasals
+  * terms        — replace_terms, MEDICAL_TERMS
+  * repetition   — collapse_repetition
+  * transcript   — normalize_transcript (batteries-included pipeline)
+
+For invisible/typographic cleanup of general (non-Devanagari) text, compose with
+``llmclean``.
+"""
+
+from .numbers import WORD_TO_NUM, parse_hindi_number, convert_numbers
+from .devanagari import (
+    strip_zero_width,
+    devanagari_digits_to_ascii,
+    ascii_digits_to_devanagari,
+    normalize_danda,
+    fold_nasals,
+    normalize_devanagari,
+)
+from .terms import MEDICAL_TERMS, replace_terms
+from .repetition import collapse_repetition
+from .transcript import normalize_transcript
+
+__all__ = [
+    "WORD_TO_NUM",
+    "parse_hindi_number",
+    "convert_numbers",
+    "strip_zero_width",
+    "devanagari_digits_to_ascii",
+    "ascii_digits_to_devanagari",
+    "normalize_danda",
+    "fold_nasals",
+    "normalize_devanagari",
+    "MEDICAL_TERMS",
+    "replace_terms",
+    "collapse_repetition",
+    "normalize_transcript",
+]
+__version__ = "0.1.0"

hindi_normalize/devanagari.py

@@ -0,0 +1,244 @@
+"""devanagari.py — character-level normalization of Devanagari (Hindi) text.
+
+Devanagari has several ways to encode the same visible text, and OCR/ASR/
+copy-paste freely mix them. Left alone, these break equality checks, search,
+regex, tokenization, and dedup — the string *looks* identical but isn't. This
+module folds the common variants to a consistent form, following the operations
+the Indic NLP Library's ``DevanagariNormalizer`` established as standard.
+
+Handled:
+
+  * **Invisible / space characters.** Zero-width joiner/non-joiner, zero-width
+    space, BOM, soft hyphen, word joiner are removed; no-break space becomes a
+    regular space. On by default.
+
+  * **Encoding variants (NFC).** क़ can be one codepoint (U+0958) or क + nukta
+    (U+0915 U+093C), and matras can arrive in non-canonical order. ``compose``
+    applies Unicode NFC so both spellings become the same bytes. Lossless.
+
+  * **Devanagari digits.** ``५`` (U+096B) vs ``5``. ``digits`` maps Devanagari
+    digits to ASCII; :func:`ascii_digits_to_devanagari` does the reverse.
+
+  * **Danda punctuation.** ASR/OCR emit ASCII ``|`` where the danda ``।``
+    belongs, repeat it, or put a space before it. ``danda`` fixes all three.
+
+  * **Colon → visarga (opt-in).** ``दु:ख`` typed with an ASCII colon becomes
+    ``दुःख``. Off by default — a colon after a Devanagari word is sometimes real
+    punctuation.
+
+  * **Nukta removal / nasal folding (opt-in, lossy).** ``remove_nukta`` folds
+    क़→क, ज़→ज; ``nasals`` folds न्, म् … clusters to anusvara (हिन्दी→हिंदी);
+    ``chandrabindu`` folds ँ→ं. All off by default.
+
+Unlike a naive ``unicodedata``-category filter, nothing here strips the Unicode
+Mark category, so matras and the virama survive.
+
+For general typographic punctuation (smart quotes, em dashes), compose this with
+``llmclean``'s ``normalize_typography`` rather than duplicating it.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+__all__ = [
+    "strip_zero_width",
+    "devanagari_digits_to_ascii",
+    "ascii_digits_to_devanagari",
+    "normalize_danda",
+    "fold_nasals",
+    "normalize_devanagari",
+]
+
+_DEVA_RANGE = "ऀ-ॿ"
+_NUKTA = "़"
+_ANUSVARA = "ं"
+_CHANDRABINDU = "ँ"
+_VISARGA = "ः"
+_DANDA = "।"
+_DOUBLE_DANDA = "॥"
+
+# ---------------------------------------------------------------------------
+# Invisibles / spaces
+# ---------------------------------------------------------------------------
+
+_ZERO_WIDTH_TRANSLATE = {
+    0xFEFF: None, 0xFFFE: None,           # BOM
+    0x200B: None, 0x200C: None, 0x200D: None,  # ZWSP, ZWNJ, ZWJ
+    0x00AD: None,                          # soft hyphen
+    0x2060: None,                          # word joiner
+    0x00A0: 0x20,                          # NBSP → space
+}
+
+
+def strip_zero_width(text: str) -> str:
+    """Remove zero-width and joiner characters; map no-break space to space.
+
+    Follows the Indic NLP base normalization: strips ZWJ/ZWNJ/ZWSP, BOM, soft
+    hyphen, and word joiner, and turns a no-break space into a regular one.
+    Note this drops ZWJ/ZWNJ unconditionally, which can affect explicit conjunct
+    rendering; for Hindi normalization that is the accepted trade-off. Returns
+    the input unchanged when nothing matches; never raises.
+    """
+    if not isinstance(text, str):
+        return text
+    try:
+        return text.translate(_ZERO_WIDTH_TRANSLATE)
+    except Exception:
+        return text
+
+
+# ---------------------------------------------------------------------------
+# Digits
+# ---------------------------------------------------------------------------
+
+_DEVA_DIGITS = "०१२३४५६७८९"
+_ASCII_DIGITS = "0123456789"
+_DEVA_TO_ASCII = {ord(d): a for d, a in zip(_DEVA_DIGITS, _ASCII_DIGITS)}
+_ASCII_TO_DEVA = {ord(a): d for d, a in zip(_DEVA_DIGITS, _ASCII_DIGITS)}
+
+
+def devanagari_digits_to_ascii(text: str) -> str:
+    """Map Devanagari digits (``०१२…``) to ASCII digits (``012…``).
+
+    Returns the input unchanged when it holds none; never raises.
+    """
+    if not isinstance(text, str):
+        return text
+    try:
+        return text.translate(_DEVA_TO_ASCII)
+    except Exception:
+        return text
+
+
+def ascii_digits_to_devanagari(text: str) -> str:
+    """Map ASCII digits (``012…``) to Devanagari digits (``०१२…``).
+
+    The inverse of :func:`devanagari_digits_to_ascii`; never raises.
+    """
+    if not isinstance(text, str):
+        return text
+    try:
+        return text.translate(_ASCII_TO_DEVA)
+    except Exception:
+        return text
+
+
+# ---------------------------------------------------------------------------
+# Danda punctuation
+# ---------------------------------------------------------------------------
+
+_PIPE_DOUBLE_RE = re.compile(r"\|\s*\|")
+_PIPE_SINGLE_RE = re.compile(r"\|")
+_DANDA_RUN_RE = re.compile(r"।(?:\s*।)+")
+_SPACE_BEFORE_DANDA_RE = re.compile(r"\s+([।॥])")
+
+
+def normalize_danda(text: str) -> str:
+    """Normalize danda punctuation.
+
+    Converts ASCII pipes used as dandas (``|``→``।``, ``||``→``॥``), collapses a
+    run of repeated dandas to one, and removes whitespace before a danda.
+    Returns the input unchanged when nothing matches; never raises.
+    """
+    if not isinstance(text, str):
+        return text
+    try:
+        text = _PIPE_DOUBLE_RE.sub(_DOUBLE_DANDA, text)
+        text = _PIPE_SINGLE_RE.sub(_DANDA, text)
+        text = _DANDA_RUN_RE.sub(_DANDA, text)
+        text = _SPACE_BEFORE_DANDA_RE.sub(r"\1", text)
+        return text
+    except Exception:
+        return text
+
+
+# ---------------------------------------------------------------------------
+# Nukta / nasals / visarga
+# ---------------------------------------------------------------------------
+
+# Pancham (class-nasal) consonant + halant before a consonant = anusvara.
+_PANCHAM = "ङञणनम"
+_NASAL_CLUSTER_RE = re.compile(r"[" + _PANCHAM + r"]्(?=[क-ह])")
+_COLON_VISARGA_RE = re.compile(r"([" + _DEVA_RANGE + r"]):")
+
+
+def fold_nasals(text: str, *, chandrabindu: bool = False) -> str:
+    """Fold explicit nasal-cluster spellings to anusvara (``ं``).
+
+    Rewrites a pancham consonant + halant before another consonant to anusvara
+    (``हिन्दी``→``हिंदी``). **Lossy** — discards which nasal was written. When
+    ``chandrabindu`` is true, also folds ``ँ``→``ं``. Never raises.
+    """
+    if not isinstance(text, str):
+        return text
+    try:
+        text = _NASAL_CLUSTER_RE.sub(_ANUSVARA, text)
+        if chandrabindu:
+            text = text.replace(_CHANDRABINDU, _ANUSVARA)
+        return text
+    except Exception:
+        return text
+
+
+def _remove_nukta(text: str) -> str:
+    text = unicodedata.normalize("NFD", text).replace(_NUKTA, "")
+    return unicodedata.normalize("NFC", text)
+
+
+# ---------------------------------------------------------------------------
+# Orchestrator
+# ---------------------------------------------------------------------------
+
+_MULTISPACE_RE = re.compile(r"[^\S\n]{2,}")
+
+
+def normalize_devanagari(
+    text: str,
+    *,
+    zero_width: bool = True,
+    compose: bool = True,
+    digits: bool = True,
+    danda: bool = True,
+    visarga: bool = False,
+    remove_nukta: bool = False,
+    nasals: bool = False,
+    chandrabindu: bool = False,
+    collapse_space: bool = True,
+) -> str:
+    """Character-level normalization of Devanagari text.
+
+    Runs the enabled steps in order: strip invisibles (``zero_width``), NFC
+    (``compose``), nukta removal (``remove_nukta``), Devanagari→ASCII digits
+    (``digits``), danda punctuation (``danda``), colon→visarga (``visarga``),
+    nasal folding (``nasals``/``chandrabindu``), and space-run collapse
+    (``collapse_space``, newlines kept).
+
+    ``zero_width``, ``compose``, ``digits``, ``danda``, ``collapse_space`` are on
+    by default (safe/lossless bar the accepted ZWJ trade-off). ``visarga``,
+    ``remove_nukta``, ``nasals``, ``chandrabindu`` are lossy and off by default.
+    Never raises.
+    """
+    if not isinstance(text, str):
+        return text
+    try:
+        if zero_width:
+            text = strip_zero_width(text)
+        if compose:
+            text = unicodedata.normalize("NFC", text)
+        if remove_nukta:
+            text = _remove_nukta(text)
+        if digits:
+            text = devanagari_digits_to_ascii(text)
+        if danda:
+            text = normalize_danda(text)
+        if visarga:
+            text = _COLON_VISARGA_RE.sub(r"\1" + _VISARGA, text)
+        if nasals or chandrabindu:
+            text = fold_nasals(text, chandrabindu=chandrabindu)
+        if collapse_space:
+            text = _MULTISPACE_RE.sub(" ", text)
+        return text
+    except Exception:
+        return text

hindi_normalize/numbers.py

@@ -0,0 +1,273 @@
+"""numbers.py — convert Hindi number words to digits.
+
+The core problem: a speech recogniser transcribes spoken Hindi numerals as
+*words* ("एक सौ दस"), not digits ("110"), and downstream code that wants a BP
+reading, a weight, or a dose needs the digits. This module maps Hindi number
+words (0-999) to integers, tolerant of the spelling variants an ASR engine
+produces ("पाँच"/"पांच"/"पाच", "सत्तर"/"सतर").
+
+    >>> convert_numbers("एक सौ दस बटा सत्तर")
+    '110 बटा 70'
+    >>> parse_hindi_number("नौ सौ निन्यानवे")
+    999
+
+A deliberate design choice: adjacent *unrelated* number words are NOT summed.
+"दो तीन दिन" ("two-three days") stays "2 3 दिन", never "5 दिन" — collapsing a
+spoken range into its sum is a classic normalization bug.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Optional, Tuple
+
+# Number words 0-100, including common ASR misspellings.
+WORD_TO_NUM: Dict[str, int] = {
+    # 0-10
+    "शून्य": 0, "एक": 1, "दो": 2, "तीन": 3, "चार": 4,
+    "पांच": 5, "पाँच": 5, "पाच": 5, "छह": 6, "छः": 6,
+    "सात": 7, "आठ": 8, "नौ": 9, "दस": 10,
+    # 11-19
+    "ग्यारह": 11, "गयारह": 11, "ग्यारा": 11,
+    "बारह": 12, "बारा": 12,
+    "तेरह": 13, "तेरा": 13,
+    "चौदह": 14, "चौदा": 14,
+    "पंद्रह": 15, "पन्द्रह": 15, "पंद्रा": 15,
+    "सोलह": 16, "सोला": 16,
+    "सत्रह": 17, "सत्तरह": 17,
+    "अठारह": 18, "अठारा": 18,
+    "उन्नीस": 19, "उन्निस": 19,
+    # 20-29
+    "बीस": 20, "इक्कीस": 21, "इक्किस": 21,
+    "बाईस": 22, "बाइस": 22,
+    "तेईस": 23, "तेइस": 23,
+    "चौबीस": 24, "चौबिस": 24,
+    "पच्चीस": 25, "पचीस": 25, "पच्चिस": 25,
+    "छब्बीस": 26, "छब्बिस": 26,
+    "सत्ताईस": 27, "सत्ताइस": 27,
+    "अट्ठाईस": 28, "अट्ठाइस": 28, "अठ्ठाईस": 28,
+    "उनतीस": 29, "उन्तीस": 29,
+    # 30-39
+    "तीस": 30, "इकतीस": 31, "इकत्तीस": 31,
+    "बत्तीस": 32, "बतीस": 32,
+    "तैंतीस": 33, "तेंतीस": 33,
+    "चौंतीस": 34, "चौतीस": 34,
+    "पैंतीस": 35, "पेंतीस": 35,
+    "छत्तीस": 36, "छतीस": 36,
+    "सैंतीस": 37, "सेंतीस": 37,
+    "अड़तीस": 38, "अडतीस": 38,
+    "उनतालीस": 39, "उन्तालीस": 39,
+    # 40-49
+    "चालीस": 40, "चालिस": 40,
+    "इकतालीस": 41, "एकतालीस": 41,
+    "बयालीस": 42, "बयालिस": 42,
+    "तैंतालीस": 43, "तेंतालीस": 43,
+    "चौवालीस": 44, "चवालीस": 44,
+    "पैंतालीस": 45, "पेंतालीस": 45,
+    "छियालीस": 46, "छयालीस": 46,
+    "सैंतालीस": 47, "सेंतालीस": 47,
+    "अड़तालीस": 48, "अडतालीस": 48,
+    "उनचास": 49,
+    # 50-59
+    "पचास": 50,
+    "इक्यावन": 51,
+    "बावन": 52,
+    "तिरपन": 53, "तिरेपन": 53,
+    "चौवन": 54, "चौबन": 54,
+    "पचपन": 55,
+    "छप्पन": 56, "छपन": 56,
+    "सत्तावन": 57, "सतावन": 57,
+    "अट्ठावन": 58, "अठावन": 58, "अठ्ठावन": 58,
+    "उनसठ": 59,
+    # 60-69
+    "साठ": 60, "साट": 60,
+    "इकसठ": 61, "एकसठ": 61,
+    "बासठ": 62, "बासट": 62,
+    "तिरसठ": 63, "तिरेसठ": 63,
+    "चौंसठ": 64, "चौसठ": 64,
+    "पैंसठ": 65, "पेंसठ": 65,
+    "छियासठ": 66, "छयासठ": 66,
+    "सड़सठ": 67, "सडसठ": 67,
+    "अड़सठ": 68, "अडसठ": 68,
+    "उनहत्तर": 69, "उनहतर": 69,
+    # 70-79
+    "सत्तर": 70, "सतर": 70,
+    "इकहत्तर": 71, "इकहतर": 71,
+    "बहत्तर": 72, "बहतर": 72,
+    "तिहत्तर": 73, "तिहतर": 73,
+    "चौहत्तर": 74, "चौहतर": 74,
+    "पचहत्तर": 75, "पचहतर": 75,
+    "छिहत्तर": 76, "छिहतर": 76,
+    "सतहत्तर": 77, "सतहतर": 77,
+    "अठहत्तर": 78, "अठहतर": 78,
+    "उन्यासी": 79, "उनासी": 79, "उन्नासी": 79,
+    # 80-89
+    "अस्सी": 80, "अस्सि": 80,
+    "इक्यासी": 81, "एक्यासी": 81,
+    "बयासी": 82, "ब्यासी": 82,
+    "तिरासी": 83,
+    "चौरासी": 84,
+    "पचासी": 85,
+    "छियासी": 86, "छयासी": 86,
+    "सत्तासी": 87, "सतासी": 87,
+    "अट्ठासी": 88, "अठासी": 88,
+    "नवासी": 89, "नव्वासी": 89,
+    # 90-99
+    "नब्बे": 90, "नब्बें": 90,
+    "इक्यानवे": 91,
+    "बानवे": 92,
+    "तिरानवे": 93,
+    "चौरानवे": 94,
+    "पंचानवे": 95, "पचानवे": 95,
+    "छियानवे": 96,
+    "सत्तानवे": 97, "सतानवे": 97,
+    "अट्ठानवे": 98, "अठानवे": 98,
+    "निन्यानवे": 99, "निन्नानवे": 99,
+    # Hundred marker
+    "सौ": 100, "सो": 100,
+}
+
+# Longest-first for greedy matching.
+_NUM_SORTED = sorted(WORD_TO_NUM.items(), key=lambda x: -len(x[0]))
+
+_DEVA = r"ऀ-ॿ"
+_NUM_WORD_INNER = r"(?:" + "|".join(re.escape(w) for w, _ in _NUM_SORTED) + r")"
+
+# A run of number words with Devanagari-aware boundaries.
+_NUM_SEQ_RE = re.compile(
+    r"(?<![" + _DEVA + r"])"
+    + _NUM_WORD_INNER
+    + r"(?:\s+" + _NUM_WORD_INNER + r")*"
+    + r"(?![" + _DEVA + r"])"
+)
+
+# ASR merges digit-word onto सौ ("एकसो" → "एक सो").
+_COMPOUND_SPLITS = re.compile(
+    r"(एकसो|दोसो|तीनसो|चारसो|पांचसो|पाँचसो|छहसो|सातसो|आठसो|नौसो)"
+)
+_COMPOUND_SPLIT_MAP = {
+    "एकसो": "एक सो", "दोसो": "दो सो", "तीनसो": "तीन सो",
+    "चारसो": "चार सो", "पांचसो": "पांच सो", "पाँचसो": "पाँच सो",
+    "छहसो": "छह सो", "सातसो": "सात सो", "आठसो": "आठ सो", "नौसो": "नौ सो",
+}
+
+
+def _parse_one_number(words: List[str], start: int) -> Tuple[int, Optional[int]]:
+    """Parse one Hindi number expression starting at ``words[start]``.
+
+    Returns ``(consumed_word_count, value)``, or ``(0, None)`` if no number
+    begins at ``start``.
+
+    Recognized patterns::
+
+        [1-9] सौ [1-99]   →  एक सौ साठ = 160
+        [1-9] सौ           →  दो सौ = 200
+        सौ [1-99]          →  सौ दस = 110
+        सौ                 →  सौ = 100
+        [0-99]             →  अट्ठावन = 58, दस = 10
+
+    Adjacent *simple* digits are NOT merged: ``"दो तीन"`` yields ``(1, 2)`` — the
+    caller advances and parses ``"तीन"`` separately.
+    """
+    n = len(words)
+    if start >= n:
+        return 0, None
+    v0 = WORD_TO_NUM.get(words[start])
+    if v0 is None:
+        return 0, None
+
+    # Pattern: [1-9] सौ [optional 1-99]
+    if 1 <= v0 < 10 and start + 1 < n and WORD_TO_NUM.get(words[start + 1]) == 100:
+        total = v0 * 100
+        if start + 2 < n:
+            v2 = WORD_TO_NUM.get(words[start + 2])
+            if v2 is not None and 0 < v2 < 100:
+                return 3, total + v2
+        return 2, total
+
+    # Pattern: सौ [optional 1-99]
+    if v0 == 100:
+        if start + 1 < n:
+            v1 = WORD_TO_NUM.get(words[start + 1])
+            if v1 is not None and 0 < v1 < 100:
+                return 2, 100 + v1
+        return 1, 100
+
+    # Pattern: any single number word (0-99)
+    return 1, v0
+
+
+def parse_hindi_number(text: str) -> Optional[int]:
+    """Parse a single Hindi number expression into an integer.
+
+    For sequences of *unrelated* numbers (e.g. ``"दो तीन"`` — two then three),
+    returns only the first parseable number (``2``). Use :func:`convert_numbers`
+    to handle mixed sequences within a real transcript.
+
+    Examples::
+
+        "एक सौ दस"       → 110
+        "एक सौ पचपन"    → 155
+        "दो सौ"           → 200
+        "सौ"              → 100
+        "सत्तर"            → 70
+        "अट्ठावन"          → 58
+        "नौ सौ निन्यानवे" → 999
+
+    Returns ``None`` for empty input or a leading non-number word. Never raises.
+    """
+    if not isinstance(text, str):
+        return None
+    try:
+        words = text.strip().split()
+        if not words:
+            return None
+        consumed, val = _parse_one_number(words, 0)
+        if consumed == 0:
+            return None
+        return val
+    except Exception:
+        return None
+
+
+def convert_numbers(text: str) -> str:
+    """Replace every Hindi number-word sequence in *text* with digit strings.
+
+    Within a matched sequence, numbers are parsed one at a time, so unrelated
+    adjacent number words (``"दो तीन"``) stay separate (``"2 3"``) instead of
+    summing.
+
+    Examples::
+
+        "एक सौ दस"            → "110"
+        "एक सौ दस बटा सत्तर"   → "110 बटा 70"
+        "अट्ठावन kg"          → "58 kg"
+        "एकसो दस"             → "110"   (compound-split ASR artifact)
+
+    Returns the input unchanged when it holds no number words; never raises.
+    """
+    if not isinstance(text, str):
+        return text
+    try:
+        # Pre-split merged compounds like "एकसो" → "एक सो".
+        text = _COMPOUND_SPLITS.sub(
+            lambda m: _COMPOUND_SPLIT_MAP.get(m.group(0), m.group(0)), text
+        )
+
+        def _replace(m: "re.Match[str]") -> str:
+            words = m.group(0).split()
+            out: List[str] = []
+            i = 0
+            while i < len(words):
+                consumed, val = _parse_one_number(words, i)
+                if consumed == 0:
+                    out.append(words[i])
+                    i += 1
+                else:
+                    out.append(str(val))
+                    i += consumed
+            return " ".join(out)
+
+        return _NUM_SEQ_RE.sub(_replace, text)
+    except Exception:
+        return text

hindi_normalize/repetition.py

@@ -0,0 +1,47 @@
+"""repetition.py — collapse inline repetition artifacts from ASR output.
+
+Speech recognisers stutter: they emit the same short cluster or word several
+times in a row (``ठीकठीकठीकठीक``, ``है है है है``). This collapses those runs
+back to a single instance.
+
+This targets *inline, back-to-back* repeats anywhere in the text. For runaway
+*tail* loops at the end of long LLM generations (repeated sentences or
+paragraphs), reach for ``llmclean``'s ``trim_repetition`` instead — the two are
+complementary.
+
+    >>> collapse_repetition("ठीकठीकठीकठीक है")
+    'ठीक है'
+    >>> collapse_repetition("हाँ हाँ हाँ हाँ जी")
+    'हाँ जी'
+"""
+
+from __future__ import annotations
+
+import re
+
+__all__ = ["collapse_repetition"]
+
+
+def collapse_repetition(text: str, *, min_repeats: int = 4) -> str:
+    """Collapse back-to-back repeats of a short cluster or whole word.
+
+    A cluster of up to 5 characters or a whitespace-separated word repeated
+    ``min_repeats`` or more times in a row collapses to one instance. Returns
+    the input unchanged when nothing repeats that often; never raises.
+
+    Parameters
+    ----------
+    text:
+        Raw ASR text.
+    min_repeats:
+        Minimum consecutive repeats before collapsing (default 4).
+    """
+    if not isinstance(text, str) or min_repeats < 2:
+        return text
+    try:
+        n = min_repeats - 1
+        text = re.sub(r"(.{1,5}?)\1{" + str(n) + r",}", r"\1", text)
+        text = re.sub(r"(\S+)(?:\s+\1){" + str(n) + r",}", r"\1", text)
+        return text
+    except Exception:
+        return text

hindi_normalize/terms.py

@@ -0,0 +1,52 @@
+"""terms.py — map spoken/mis-recognized terms to canonical forms.
+
+An ASR engine writes English clinical or technical terms inconsistently: some
+in Latin (``BP``), some transliterated into Devanagari (``बीपी``, ``बीबी``),
+some spelled several different ways. A small term dictionary maps every variant
+to one canonical form.
+
+:data:`MEDICAL_TERMS` ships a maternal/child-health (ASHA home-visit) vocabulary
+as a ready default. Pass your own dictionary to :func:`replace_terms` for any
+other domain — the replacement logic is domain-agnostic.
+
+    >>> replace_terms("बीबी एक सौ दस बटा सत्तर", MEDICAL_TERMS)
+    'BP एक सौ दस / सत्तर'
+"""
+
+from __future__ import annotations
+
+from typing import Dict, Mapping
+
+__all__ = ["MEDICAL_TERMS", "replace_terms"]
+
+MEDICAL_TERMS: Dict[str, str] = {
+    "बीपी": "BP", "भीपी": "BP", "बीबी": "BP", "बी पी": "BP", "बी.पी.": "BP",
+    "एचबी": "Hb", "हबी": "Hb", "हीमोग्लोबिन": "Hb", "एच बी": "Hb",
+    "आईएफए": "IFA", "आई एफ ए": "IFA",
+    "टीटी": "TT", "टी टी": "TT",
+    "पीएचसी": "PHC", "पी एच सी": "PHC", "पीएचसे": "PHC",
+    "सीएचसी": "CHC", "सी एच सी": "CHC",
+    "बीसीजी": "BCG", "ओपीवी": "OPV", "हेप बी": "Hep-B",
+    "आईएमएनसीआई": "IMNCI",
+    "किलो": "kg", "किलोग्राम": "kg",
+    "बटा": "/", "बता": "/",
+    "दशमलव": ".", "दशम्लव": ".", "दशम्लफ": ".",
+    "डिग्री": "°",
+}
+
+
+def replace_terms(text: str, terms: Mapping[str, str] = MEDICAL_TERMS) -> str:
+    """Replace each key of *terms* in *text* with its value, longest key first.
+
+    Longest-first ordering stops a short key from firing inside a longer one.
+    Pass ``terms=None`` or ``{}`` to no-op. Returns the input unchanged when
+    nothing matches; never raises.
+    """
+    if not isinstance(text, str) or not terms:
+        return text
+    try:
+        for spoken, canonical in sorted(terms.items(), key=lambda x: -len(x[0])):
+            text = text.replace(spoken, canonical)
+        return text
+    except Exception:
+        return text

hindi_normalize/transcript.py

@@ -0,0 +1,74 @@
+"""transcript.py — batteries-included normalization for Hindi ASR transcripts.
+
+:func:`normalize_transcript` chains the package's transforms in the order that
+works for real speech-to-text output — collapse stutter, canonicalize the
+Devanagari, map domain terms, convert number words to digits, tidy the
+punctuation — so a raw Whisper/ASR string comes out ready for downstream
+extraction:
+
+    >>> normalize_transcript("आपका BP एक सौ दस बटा सत्तर है, वजन अट्ठावन kg")
+    'आपका BP 110/70 है, वजन 58 kg'
+
+Each stage is a flag, and the term dictionary is swappable, so the pipeline
+adapts to non-medical domains or to callers that only want part of it.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Mapping, Optional
+
+from .devanagari import normalize_devanagari
+from .numbers import convert_numbers
+from .repetition import collapse_repetition
+from .terms import MEDICAL_TERMS, replace_terms
+
+__all__ = ["normalize_transcript"]
+
+_SLASH_RE = re.compile(r"\s*/\s*")
+_DECIMAL_RE = re.compile(r"(\d)\s*\.\s*(\d)")
+_SENTENCE_RE = re.compile(r"[।](?:\s+)")
+
+
+def normalize_transcript(
+    transcript: str,
+    *,
+    terms: Optional[Mapping[str, str]] = MEDICAL_TERMS,
+    numbers: bool = True,
+    devanagari: bool = True,
+    repetition: bool = True,
+    sentence_breaks: bool = True,
+) -> str:
+    """Normalize a Hindi ASR transcript end to end.
+
+    Pipeline (each stage toggleable):
+
+    1. **repetition** — collapse ASR stutter (:func:`collapse_repetition`).
+    2. **devanagari** — invisibles, NFC, Devanagari→ASCII digits, danda
+       (:func:`normalize_devanagari`, safe defaults only).
+    3. **terms** — map domain terms to canonical forms (:func:`replace_terms`);
+       defaults to :data:`MEDICAL_TERMS`, pass ``None`` to skip.
+    4. **numbers** — Hindi number words → digits (:func:`convert_numbers`).
+    5. Spacing cleanup around ``/`` and decimal points.
+    6. **sentence_breaks** — newline after each danda ``।``.
+
+    Trailing punctuation and whitespace are always trimmed. Never raises.
+    """
+    if not isinstance(transcript, str):
+        return transcript
+    try:
+        if repetition:
+            transcript = collapse_repetition(transcript)
+        if devanagari:
+            transcript = normalize_devanagari(transcript)
+        if terms:
+            transcript = replace_terms(transcript, terms)
+        if numbers:
+            transcript = convert_numbers(transcript)
+        transcript = _SLASH_RE.sub("/", transcript)
+        transcript = _DECIMAL_RE.sub(r"\1.\2", transcript)
+        if sentence_breaks:
+            transcript = _SENTENCE_RE.sub("।\n", transcript)
+        return transcript.strip().rstrip(",. ")
+    except Exception:
+        return transcript

hindi_normalize-0.1.0.dist-info/METADATA

@@ -0,0 +1,150 @@
+Metadata-Version: 2.4
+Name: hindi-normalize
+Version: 0.1.0
+Summary: Convert Hindi number words to digits and normalize Devanagari ASR/OCR text
+Project-URL: Homepage, https://github.com/Tushar-9802/hindi-normalize
+Project-URL: Issues, https://github.com/Tushar-9802/hindi-normalize/issues
+Author-email: Tushar Jaju <tusharbrisingr9802@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Tushar Jaju
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: asr,devanagari,hindi,indic,nlp,normalization,number-words,speech-to-text,text-processing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: Hindi
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# hindi-normalize
+
+Convert Hindi number words to digits and normalize Devanagari text from ASR, OCR, or copy-paste. Handles the encoding variants, punctuation quirks, and number-word spellings that break search, equality checks, tokenization, and downstream extraction. Zero runtime dependencies — pure standard library.
+
+I built this because [Sakhi](https://github.com/Tushar-9802/Sakhi) — an offline medical Hindi voice-to-form tool meant for ASHAs— kept reinventing the same regex passes against Whisper output: number words that never became digits, `५` and `5` treated as different characters, `|` where a danda `।` belonged, the same word encoded two different ways. This is that pass, extracted and generalized.
+
+## Install
+
+```bash
+pip install hindi-normalize
+```
+
+## Quick start
+
+```python
+from hindi_normalize import normalize_transcript
+
+normalize_transcript("आपका BP एक सौ दस बटा सत्तर है, वजन अट्ठावन kg")
+# 'आपका BP 110/70 है, वजन 58 kg'
+```
+
+`normalize_transcript` runs the whole pipeline. The individual transforms are also exported and usable on their own.
+
+## Numbers
+
+Hindi number words → digits, tolerant of ASR spelling variants (`पाँच`/`पांच`/`पाच`, `सत्तर`/`सतर`). Adjacent unrelated numbers are not summed — `दो तीन दिन` ("two-three days") stays `2 3 दिन`, never `5`.
+
+```python
+from hindi_normalize import convert_numbers, parse_hindi_number
+
+convert_numbers("एक सौ दस बटा सत्तर")   # '110 बटा 70'
+parse_hindi_number("नौ सौ निन्यानवे")    # 999
+```
+
+## Devanagari
+
+`normalize_devanagari` folds the character-level variants OCR/ASR/copy-paste introduce, following the operations the [Indic NLP Library](https://github.com/anoopkunchukuttan/indic_nlp_library) established as standard.
+
+```python
+from hindi_normalize import normalize_devanagari
+
+normalize_devanagari("वजन ५८ किलो | ठीक है")   # 'वजन 58 किलो। ठीक है'
+```
+
+Default (lossless, on):
+
+- **Invisibles** — strip ZWJ/ZWNJ/ZWSP, BOM, soft hyphen, word joiner; no-break space → space
+- **NFC** — unify nukta encodings (`क़` as one codepoint vs `क` + nukta) and matra order
+- **Digits** — Devanagari digits (`५`) → ASCII (`5`)
+- **Danda** — `|` → `।`, `||` → `॥`, collapse runs, drop space before a danda
+
+Opt-in (lossy, off):
+
+- `remove_nukta` — `क़` → `क`, `ज़` → `ज`
+- `nasals` — fold nasal clusters to anusvara (`हिन्दी` → `हिंदी`)
+- `chandrabindu` — `ँ` → `ं`
+- `visarga` — ASCII colon after a Devanagari letter → visarga (`दु:ख` → `दुःख`)
+
+Unlike a naive `unicodedata`-category filter, nothing here strips the Unicode Mark category, so matras and the virama survive — the [failure mode](https://kavyamanohar.com/post/indic-normalizer/) that silently deletes vowel signs from Indic text.
+
+## Terms
+
+Map spoken/mis-recognized terms to canonical forms. Ships a maternal/child-health (ASHA home-visit) dictionary; pass your own for any domain.
+
+```python
+from hindi_normalize import replace_terms, MEDICAL_TERMS
+
+replace_terms("बीबी एक सौ दस बटा सत्तर", MEDICAL_TERMS)   # 'BP एक सौ दस / सत्तर'
+replace_terms("क ख", {"क": "K", "ख": "KH"})              # 'K KH'
+```
+
+## Repetition
+
+Collapse the back-to-back stutter recognisers emit.
+
+```python
+from hindi_normalize import collapse_repetition
+
+collapse_repetition("ठीकठीकठीकठीक है")   # 'ठीक है'
+```
+
+For runaway *tail* loops in long LLM generations (repeated sentences/paragraphs) and for general invisible/typographic cleanup of non-Devanagari text, compose with [llmclean](https://github.com/Tushar-9802/llmclean).
+
+## API
+
+| Function | Purpose |
+|---|---|
+| `normalize_transcript(text, *, terms, numbers, devanagari, repetition, sentence_breaks)` | Full pipeline |
+| `convert_numbers(text)` | Hindi number words → digits |
+| `parse_hindi_number(text)` | Parse one number expression → `int` |
+| `normalize_devanagari(text, *, ...)` | Character-level normalization |
+| `strip_zero_width(text)` | Remove invisibles; NBSP → space |
+| `devanagari_digits_to_ascii(text)` / `ascii_digits_to_devanagari(text)` | Digit conversion |
+| `normalize_danda(text)` | Danda punctuation |
+| `fold_nasals(text, *, chandrabindu)` | Nasal clusters → anusvara |
+| `replace_terms(text, terms)` | Term dictionary replacement |
+| `collapse_repetition(text, *, min_repeats)` | Collapse inline stutter |
+
+Every function is guarded against non-string input and returns the input unchanged rather than raising.
+
+## License
+
+MIT

hindi_normalize-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+hindi_normalize/__init__.py,sha256=a-aHUhIZ9nX6wuPP0DGax_W6t0aOgOuKdCpWeNIqU3o,2041
+hindi_normalize/devanagari.py,sha256=RrpgZ7zeeaeEe2a6i-Si5tZ5mQoebPDnDtIu5k6E3N4,8766
+hindi_normalize/numbers.py,sha256=STkWUrz5f_IhAbhyG9RFRmtWxZTpsXfZp1Bc9e3XnP4,11228
+hindi_normalize/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hindi_normalize/repetition.py,sha256=mdVSBuVc5nkDnHrF0W5SxG3QsQq2NjaoYvjmQXbit_0,1666
+hindi_normalize/terms.py,sha256=noOFX9KLgT2bDJgF2DBvY6T-8_6eDksHeMbJuFCCCe8,2343
+hindi_normalize/transcript.py,sha256=1As2OlMyz91SFWNAnhaO29uCLaG1xFpFao-SB6m77kQ,2818
+hindi_normalize-0.1.0.dist-info/METADATA,sha256=te68zEuwcXZYe_Oytc6veB9DRDTr0PwHi_h2uHtrbQw,7246
+hindi_normalize-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+hindi_normalize-0.1.0.dist-info/licenses/LICENSE,sha256=q3Y2X5F4Wb1XM3ApR2m8wHWa5GeDNP0z8Wi9HqnBUxk,1068
+hindi_normalize-0.1.0.dist-info/RECORD,,

hindi_normalize-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

hindi_normalize-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tushar Jaju
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.