telugu-language-tools 4.0.2__py3-none-any.whl

telugu_lib/iso15919_mappings.py

@@ -0,0 +1,430 @@
+"""
+ISO 15919 Standard Compliant Telugu Mappings
+=============================================
+
+International standard for romanization of Indic scripts.
+Supports both diacritic notation and ASCII alternatives.
+
+Usage:
+    from telugu_lib.iso15919_mappings import get_iso_consonants, get_iso_vowels
+    
+Reference: ISO 15919:2001 - Transliteration of Devanagari and related Indic scripts
+"""
+
+# ============================================================================
+# ISO 15919 CONSONANT MAPPINGS
+# ============================================================================
+
+def get_iso_consonants(mode="mixed"):
+    """
+    Get ISO 15919 compliant consonant mappings.
+    
+    Args:
+        mode: "diacritic" (only diacritics), "ascii" (only capitals), 
+              "mixed" (both, default)
+    
+    Returns:
+        Dictionary of romanization → Telugu mappings
+    """
+    
+    # Base mappings with diacritics (ISO 15919 standard)
+    diacritic_consonants = {
+        # Velars (కవర్గ)
+        "k": "క",      # ka
+        "kh": "ఖ",     # kha
+        "g": "గ",      # ga
+        "gh": "ఘ",     # gha
+        "ṅ": "ఙ",      # ṅa (velar nasal, rare)
+        
+        # Palatals (చవర్గ)
+        "c": "చ",      # ca (ISO uses 'c', not 'ch')
+        "ch": "ఛ",     # cha (aspirated)
+        "j": "జ",      # ja
+        "jh": "ఝ",     # jha
+        "ñ": "ఞ",      # ña (palatal nasal, rare)
+        
+        # Retroflexes (టవర్గ) - with underdots
+        "ṭ": "ట",      # ṭa
+        "ṭh": "ఠ",     # ṭha
+        "ḍ": "డ",      # ḍa
+        "ḍh": "ఢ",     # ḍha
+        "ṇ": "ణ",      # ṇa (retroflex nasal)
+        
+        # Dentals (తవర్గ)
+        "t": "త",      # ta
+        "th": "థ",     # tha
+        "d": "ద",      # da
+        "dh": "ధ",     # dha
+        "n": "న",      # na (dental nasal)
+        
+        # Labials (పవర్గ)
+        "p": "ప",      # pa
+        "ph": "ఫ",     # pha
+        "b": "బ",      # ba
+        "bh": "భ",     # bha
+        "m": "మ",      # ma
+        
+        # Sonorants (అంతస్థలు)
+        "y": "య",      # ya
+        "r": "ర",      # ra
+        "l": "ల",      # la
+        "v": "వ",      # va
+        "w": "వ",      # wa (alternative for v)
+        
+        # Sibilants (ఊష్మలు)
+        "ś": "శ",      # śa (palatal sibilant)
+        "ṣ": "ష",      # ṣa (retroflex sibilant)
+        "s": "స",      # sa (dental sibilant)
+        
+        # Glottal
+        "h": "హ",      # ha
+        
+        # Additional consonants
+        "ḷ": "ళ",      # ḷa (retroflex lateral)
+        "ḻ": "ఴ",      # ḻa (Tamil retroflex, obsolete in Telugu)
+        "ṟ": "ఱ",      # ṟa (alveolar trill, archaic)
+        
+        # Simplified alternatives (common usage)
+        "sha": "శ",    # Alternative for ś
+        "Sha": "ష",    # Alternative for ṣ (capital S)
+        "za": "జ",     # z often mapped to ja
+        "f": "ఫ",      # f → pha
+    }
+    
+    # ASCII alternatives (using capitals for retroflexes)
+    ascii_consonants = {
+        # Retroflexes (capital = retroflex)
+        "T": "ట",      # ASCII for ṭ
+        "Th": "ఠ",     # ASCII for ṭh
+        "D": "డ",      # ASCII for ḍ
+        "Dh": "ఢ",     # ASCII for ḍh
+        "N": "ణ",      # ASCII for ṇ (retroflex nasal)
+        "L": "ళ",      # ASCII for ḷ (retroflex lateral)
+        "R": "ఱ",      # ASCII for ṟ (rare)
+        
+        # Sibilants
+        "S": "ష",      # ASCII for ṣ (retroflex sibilant)
+        "sh": "శ",     # Palatal sibilant (lowercase)
+        
+        # Palatals
+        "ch": "చ",     # Common ch → ca
+        "chh": "ఛ",    # Aspirated
+        
+        # Nasals
+        "ng": "ఙ",     # ASCII for ṅ (velar nasal)
+        "ny": "ఞ",     # ASCII for ñ (palatal nasal)
+    }
+    
+    # Combined mapping based on mode
+    if mode == "diacritic":
+        return diacritic_consonants
+    elif mode == "ascii":
+        return {**diacritic_consonants, **ascii_consonants}
+    else:  # mixed (default)
+        return {**diacritic_consonants, **ascii_consonants}
+
+
+# ============================================================================
+# ISO 15919 VOWEL MAPPINGS
+# ============================================================================
+
+def get_iso_vowels(mode="mixed"):
+    """
+    Get ISO 15919 compliant vowel mappings.
+    
+    Args:
+        mode: "diacritic" (only diacritics), "ascii" (only capitals), 
+              "mixed" (both, default)
+    
+    Returns:
+        Dictionary of romanization → Telugu vowel mappings
+    """
+    
+    # Base vowels with diacritics (ISO 15919 standard)
+    diacritic_vowels = {
+        # Short vowels
+        "a": "అ",      # a (short)
+        "i": "ఇ",      # i (short)
+        "u": "ఉ",      # u (short)
+        "ṛ": "ఋ",      # ṛ (vocalic r, short)
+        "ḷ": "ఌ",      # ḷ (vocalic l, short, very rare)
+        
+        # Long vowels (with macrons)
+        "ā": "ఆ",      # ā (long)
+        "ī": "ఈ",      # ī (long)
+        "ū": "ఊ",      # ū (long)
+        "ṝ": "ౠ",      # ṝ (vocalic r, long)
+        "ḹ": "ౡ",      # ḹ (vocalic l, long, very rare)
+        
+        # E vowels
+        "e": "ఎ",      # e (short)
+        "ē": "ఏ",      # ē (long)
+        
+        # O vowels
+        "o": "ఒ",      # o (short)
+        "ō": "ఓ",      # ō (long)
+        
+        # Diphthongs
+        "ai": "ఐ",     # ai
+        "au": "ఔ",     # au
+        
+        # Special markers
+        "ṁ": "ం",      # ṁ (anusvara)
+        "ḥ": "ః",      # ḥ (visarga)
+        "m̐": "ఁ",      # candrabindu (rare)
+    }
+    
+    # ASCII alternatives
+    ascii_vowels = {
+        # Long vowels (capital = long, or double letter)
+        "A": "ఆ",      # ASCII for ā
+        "aa": "ఆ",     # Alternative for ā
+        "I": "ఈ",      # ASCII for ī
+        "ii": "ఈ",     # Alternative for ī
+        "U": "ఊ",      # ASCII for ū
+        "uu": "ఊ",     # Alternative for ū
+        "E": "ఏ",      # ASCII for ē (long e)
+        "ee": "ఏ",     # Alternative for ē
+        "O": "ఓ",      # ASCII for ō (long o)
+        "oo": "ఓ",     # Alternative for ō
+        
+        # Vocalic consonants
+        "R": "ఋ",      # ASCII for ṛ
+        "ri": "ఋ",     # Common alternative
+        "RR": "ౠ",     # ASCII for ṝ (long)
+        "rii": "ౠ",    # Common alternative
+        
+        # Vocalic l (very rare)
+        "lR": "ఌ",     # ASCII for ḷ
+        "li": "ఌ",     # Common alternative
+        
+        # Special markers
+        "M": "ం",      # ASCII for ṁ (anusvara)
+        "am": "ం",     # Common representation
+        "H": "ః",      # ASCII for ḥ (visarga)
+        "ah": "ః",     # Common representation
+    }
+    
+    if mode == "diacritic":
+        return diacritic_vowels
+    elif mode == "ascii":
+        return {**diacritic_vowels, **ascii_vowels}
+    else:  # mixed
+        return {**diacritic_vowels, **ascii_vowels}
+
+
+# ============================================================================
+# ISO 15919 MATRA (VOWEL SIGN) MAPPINGS
+# ============================================================================
+
+def get_iso_matras(mode="mixed"):
+    """
+    Get ISO 15919 compliant matra (vowel sign) mappings.
+    
+    Matras are vowel signs that attach to consonants.
+    
+    Returns:
+        Dictionary of romanization → Telugu matra mappings
+    """
+    
+    diacritic_matras = {
+        # No marking for inherent 'a'
+        "a": "",       # Inherent vowel (no mark)
+        
+        # Short vowel signs
+        "i": "ి",      # i-matra
+        "u": "ు",      # u-matra
+        "ṛ": "ృ",      # ṛ-matra (vocalic r)
+        "ḷ": "ౢ",      # ḷ-matra (vocalic l, rare)
+        
+        # Long vowel signs
+        "ā": "ా",      # ā-matra
+        "ī": "ీ",      # ī-matra
+        "ū": "ూ",      # ū-matra
+        "ṝ": "ౄ",      # ṝ-matra (long vocalic r)
+        "ḹ": "ౣ",      # ḹ-matra (long vocalic l, rare)
+        
+        # E vowel signs
+        "e": "ె",      # e-matra (short)
+        "ē": "ే",      # ē-matra (long)
+        
+        # O vowel signs
+        "o": "ొ",      # o-matra (short)
+        "ō": "ో",      # ō-matra (long)
+        
+        # Diphthong signs
+        "ai": "ై",     # ai-matra
+        "au": "ౌ",     # au-matra
+        
+        # Special markers (same as standalone)
+        "ṁ": "ం",      # anusvara
+        "ḥ": "ః",      # visarga
+    }
+    
+    ascii_matras = {
+        # ASCII alternatives for long vowels
+        "A": "ా",      # ASCII for ā
+        "aa": "ా",     # Common alternative
+        "I": "ీ",      # ASCII for ī
+        "ii": "ీ",     # Common alternative
+        "U": "ూ",      # ASCII for ū
+        "uu": "ూ",     # Common alternative
+        "E": "ే",      # ASCII for ē
+        "ee": "ే",     # Common alternative
+        "O": "ో",      # ASCII for ō
+        "oo": "ో",     # Common alternative
+        
+        # Vocalic consonants
+        "R": "ృ",      # ASCII for ṛ
+        "ri": "ృ",     # Common alternative
+        "RR": "ౄ",     # ASCII for ṝ
+        "rii": "ౄ",    # Common alternative
+        
+        # Special markers
+        "M": "ం",      # ASCII for ṁ
+        "am": "ం",     # Common representation
+        "H": "ః",      # ASCII for ḥ
+        "ah": "ః",     # Common representation
+    }
+    
+    if mode == "diacritic":
+        return diacritic_matras
+    elif mode == "ascii":
+        return {**diacritic_matras, **ascii_matras}
+    else:  # mixed
+        return {**diacritic_matras, **ascii_matras}
+
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+def normalize_input(text):
+    """
+    Normalize input text to handle common variations.
+    
+    Converts common romanization variants to ISO 15919 standard.
+    """
+    replacements = {
+        # Common variations → ISO standard
+        "Ch": "ch",    # Capital Ch → ch
+        "zh": "j",     # zh → j
+        "Z": "j",      # Z → j
+        "ph": "ph",    # Already correct
+        "f": "ph",     # f → ph (no native f in Telugu)
+        "q": "k",      # q → k (no native q)
+        "x": "ks",     # x → ks cluster
+        
+        # Ensure ASCII capitals for retroflexes are preserved
+        # (handled separately in get_iso_consonants)
+    }
+    
+    result = text
+    for old, new in replacements.items():
+        result = result.replace(old, new)
+    
+    return result
+
+
+def get_articulation_class(consonant):
+    """
+    Get the articulation class of a consonant for nasal assimilation.
+    
+    Returns:
+        String: "velar", "palatal", "retroflex", "dental", "labial", or None
+    """
+    VELAR = ["k", "kh", "g", "gh", "ṅ", "ng"]
+    PALATAL = ["c", "ch", "chh", "j", "jh", "ñ", "ny", "ś", "sh"]
+    RETROFLEX = ["ṭ", "ṭh", "ḍ", "ḍh", "ṇ", "ṣ", "T", "Th", "D", "Dh", "N", "S", "ḷ", "L"]
+    DENTAL = ["t", "th", "d", "dh", "n", "s"]
+    LABIAL = ["p", "ph", "b", "bh", "m", "v", "w"]
+    
+    if consonant in VELAR:
+        return "velar"
+    elif consonant in PALATAL:
+        return "palatal"
+    elif consonant in RETROFLEX:
+        return "retroflex"
+    elif consonant in DENTAL:
+        return "dental"
+    elif consonant in LABIAL:
+        return "labial"
+    else:
+        return None
+
+
+def is_retroflex(char):
+    """Check if character is a retroflex consonant"""
+    retroflexes = ["ṭ", "ṭh", "ḍ", "ḍh", "ṇ", "ṣ", "ḷ", "ṟ",
+                   "T", "Th", "D", "Dh", "N", "S", "L", "R",
+                   "ట", "ఠ", "డ", "ఢ", "ణ", "ష", "ళ", "ఱ"]
+    return char in retroflexes
+
+
+def is_dental(char):
+    """Check if character is a dental consonant"""
+    dentals = ["t", "th", "d", "dh", "n", "s",
+               "త", "థ", "ద", "ధ", "న", "స"]
+    return char in dentals
+
+
+# ============================================================================
+# VALIDATION AND TESTING
+# ============================================================================
+
+def validate_iso_mappings():
+    """Validate that all ISO 15919 standard characters are mapped"""
+    consonants = get_iso_consonants("mixed")
+    vowels = get_iso_vowels("mixed")
+    matras = get_iso_matras("mixed")
+    
+    print("ISO 15919 Mappings Validation")
+    print("=" * 50)
+    print(f"Consonants: {len(consonants)} mappings")
+    print(f"Vowels: {len(vowels)} mappings")
+    print(f"Matras: {len(matras)} mappings")
+    print(f"Total: {len(consonants) + len(vowels) + len(matras)} mappings")
+    
+    # Check for duplicates
+    all_roman = list(consonants.keys()) + list(vowels.keys())
+    duplicates = [x for x in all_roman if all_roman.count(x) > 1]
+    if duplicates:
+        print(f"\n⚠️  Warning: Duplicate roman keys: {set(duplicates)}")
+    else:
+        print("\n✅ No duplicate keys")
+    
+    # Check Telugu coverage
+    telugu_chars = set(consonants.values()) | set(vowels.values())
+    print(f"\n✅ Covers {len(telugu_chars)} unique Telugu characters")
+    
+    return True
+
+
+if __name__ == "__main__":
+    # Run validation
+    validate_iso_mappings()
+    
+    # Example usage
+    print("\n" + "=" * 50)
+    print("Example Usage:")
+    print("=" * 50)
+    
+    consonants = get_iso_consonants("mixed")
+    vowels = get_iso_vowels("mixed")
+    
+    examples = [
+        ("k", "Velar"),
+        ("ṭ", "Retroflex (diacritic)"),
+        ("T", "Retroflex (ASCII)"),
+        ("ṅ", "Velar nasal (diacritic)"),
+        ("ng", "Velar nasal (ASCII)"),
+        ("ā", "Long vowel (diacritic)"),
+        ("A", "Long vowel (ASCII)"),
+        ("aa", "Long vowel (double)"),
+    ]
+    
+    for roman, description in examples:
+        telugu_cons = consonants.get(roman)
+        telugu_vow = vowels.get(roman)
+        telugu = telugu_cons or telugu_vow or "N/A"
+        print(f"{roman:4} → {telugu:2}  ({description})")

telugu_lib/sentence_tools.py

@@ -0,0 +1,214 @@
+"""
+Sentence similarity and correction tools for Telugu text.
+
+This module provides functionality to find similar Telugu sentences
+and correct grammar/spelling using SentenceTransformers.
+"""
+
+try:
+    from sentence_transformers import SentenceTransformer, util
+    import torch
+    SENTENCE_TRANSFORMERS_AVAILABLE = True
+except ImportError:
+    SENTENCE_TRANSFORMERS_AVAILABLE = False
+
+# Preload the model (lightweight multilingual model)
+_MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2"
+_model = None
+
+
+def _get_model():
+    """
+    Lazy load the sentence transformer model.
+    Returns the cached model or loads it if not already loaded.
+    """
+    global _model
+    if _model is None:
+        if not SENTENCE_TRANSFORMERS_AVAILABLE:
+            raise ImportError(
+                "sentence-transformers is not installed. "
+                "Please install it with: pip install sentence-transformers"
+            )
+        _model = SentenceTransformer(_MODEL_NAME)
+    return _model
+
+
+def find_similar_sentence(query, reference_list, top_k=1, min_score=0.5):
+    """
+    Find the most similar sentence(s) from a list of references.
+
+    Args:
+        query (str): The query sentence in Telugu
+        reference_list (list): List of Telugu reference sentences
+        top_k (int): Number of top similar sentences to return (default: 1)
+        min_score (float): Minimum similarity score threshold (default: 0.5)
+
+    Returns:
+        tuple: (best_sentence, similarity_score) if top_k=1
+        list: List of tuples [(sentence, score), ...] if top_k > 1
+
+    Example:
+        >>> refs = ["వర్షం పడుతోంది", "ఇప్పుడు వాన వస్తోంది", "నేను తినడానికి వెళ్తున్నాను"]
+        >>> sentence, score = find_similar_sentence("వర్షం కురుస్తోంది", refs)
+        >>> print(sentence, score)
+    """
+    if not SENTENCE_TRANSFORMERS_AVAILABLE:
+        raise ImportError(
+            "sentence-transformers is required for this feature. "
+            "Install it with: pip install sentence-transformers"
+        )
+
+    if not reference_list:
+        return (None, 0.0) if top_k == 1 else []
+
+    model = _get_model()
+
+    # Encode the query and reference sentences
+    query_emb = model.encode(query, convert_to_tensor=True)
+    ref_emb = model.encode(reference_list, convert_to_tensor=True)
+
+    # Compute cosine similarity
+    scores = util.cos_sim(query_emb, ref_emb)[0]
+
+    # Find top-k most similar sentences
+    top_results = []
+    for score, sentence in sorted(zip(scores, reference_list), reverse=True)[:top_k]:
+        if float(score) >= min_score:
+            top_results.append((sentence, float(score)))
+
+    if not top_results:
+        # Return the best match even if below threshold
+        best_idx = torch.argmax(scores).item()
+        best_score = float(scores[best_idx])
+        if top_k == 1:
+            return (reference_list[best_idx], best_score)
+        else:
+            return [(reference_list[best_idx], best_score)]
+
+    if top_k == 1:
+        return (top_results[0][0], top_results[0][1])
+    return top_results
+
+
+def correct_sentence(query, references, min_score=0.5):
+    """
+    Correct a Telugu sentence by finding the best matching reference.
+
+    Args:
+        query (str): The potentially incorrect Telugu sentence
+        references (list): List of correct Telugu sentences to match against
+        min_score (float): Minimum similarity score threshold
+
+    Returns:
+        tuple: (corrected_sentence, similarity_score)
+
+    Example:
+        >>> refs = ["నేను ఇంటికి వెళ్తున్నాను", "వర్షం పడుతోంది", "ఇది మంచి పుస్తకం"]
+        >>> corrected, score = correct_sentence("వర్షం పడుతునది", refs)
+        >>> print(corrected, score)
+    """
+    return find_similar_sentence(query, references, top_k=1, min_score=min_score)
+
+
+def rank_sentences(query, reference_list, min_score=0.3):
+    """
+    Rank all reference sentences by similarity to the query.
+
+    Args:
+        query (str): The query sentence in Telugu
+        reference_list (list): List of Telugu reference sentences
+        min_score (float): Minimum similarity score to include in results
+
+    Returns:
+        list: Sorted list of tuples [(sentence, score), ...] in descending order
+
+    Example:
+        >>> refs = ["వర్షం పడుతోంది", "ఇప్పుడు వాన వస్తోంది", "నేను తినడానికి వెళ్తున్నాను"]
+        >>> ranked = rank_sentences("వర్షం కురుస్తోంది", refs)
+        >>> for sentence, score in ranked:
+        ...     print(f"{sentence}: {score:.3f}")
+    """
+    if not SENTENCE_TRANSFORMERS_AVAILABLE:
+        raise ImportError(
+            "sentence-transformers is required for this feature. "
+            "Install it with: pip install sentence-transformers"
+        )
+
+    if not reference_list:
+        return []
+
+    model = _get_model()
+
+    # Encode all sentences
+    query_emb = model.encode(query, convert_to_tensor=True)
+    ref_emb = model.encode(reference_list, convert_to_tensor=True)
+
+    # Compute similarities
+    scores = util.cos_sim(query_emb, ref_emb)[0]
+
+    # Create and sort results
+    results = [(ref, float(score))
+               for ref, score in zip(reference_list, scores)
+               if float(score) >= min_score]
+
+    # Sort by score in descending order
+    results.sort(key=lambda x: x[1], reverse=True)
+
+    return results
+
+
+def batch_similarity(queries, reference_list, batch_size=32):
+    """
+    Compute similarity for multiple queries against the reference list.
+
+    Args:
+        queries (list): List of query sentences
+        reference_list (list): List of reference sentences
+        batch_size (int): Batch size for encoding (default: 32)
+
+    Returns:
+        list: List of tuples (query, best_match, best_score) for each query
+
+    Example:
+        >>> queries = ["వర్షం కురుస్తోంది", "నేను వస్తున్నాను"]
+        >>> refs = ["వర్షం పడుతోంది", "నేను ఇంటికి వెళ్తున్నాను"]
+        >>> results = batch_similarity(queries, refs)
+        >>> for query, match, score in results:
+        ...     print(f"{query} -> {match} ({score:.3f})")
+    """
+    if not SENTENCE_TRANSFORMERS_AVAILABLE:
+        raise ImportError(
+            "sentence-transformers is required for this feature. "
+            "Install it with: pip install sentence-transformers"
+        )
+
+    if not queries or not reference_list:
+        return []
+
+    model = _get_model()
+
+    # Encode all queries
+    query_embeddings = model.encode(queries, convert_to_tensor=True, batch_size=batch_size)
+    ref_embeddings = model.encode(reference_list, convert_to_tensor=True, batch_size=batch_size)
+
+    # Compute similarities
+    similarity_matrix = util.cos_sim(query_embeddings, ref_embeddings)
+
+    results = []
+    for i, query in enumerate(queries):
+        scores = similarity_matrix[i]
+        best_idx = torch.argmax(scores).item()
+        best_score = float(scores[best_idx])
+        results.append((query, reference_list[best_idx], best_score))
+
+    return results
+
+
+def is_sentence_transformers_available():
+    """
+    Check if sentence-transformers library is available.
+
+    Returns:
+        bool: True if sentence-transformers is installed, False otherwise
+    """
+    return SENTENCE_TRANSFORMERS_AVAILABLE