kannada-tokenizer 0.1.0__py3-none-any.whl

kannada_tokenizer/__init__.py

@@ -0,0 +1,6 @@
+"""Kannada tokenizer with sandhi splitting for Information Retrieval."""
+
+from kannada_tokenizer.tokenizer import tokenize
+
+__version__ = "0.1.0"
+__all__ = ["tokenize"]

kannada_tokenizer/cli.py

@@ -0,0 +1,65 @@
+"""Command-line interface for the Kannada tokenizer."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from .tokenizer import tokenize, tokenize_words
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    """Build and return the argument parser."""
+    parser = argparse.ArgumentParser(
+        prog="kannada-tokenize",
+        description="Tokenize Kannada text with optional sandhi splitting.",
+    )
+    parser.add_argument(
+        "text",
+        nargs="?",
+        default=None,
+        help="Kannada text to tokenize. Reads from stdin if omitted.",
+    )
+    parser.add_argument(
+        "--no-sandhi",
+        action="store_true",
+        default=False,
+        help="Disable sandhi splitting (word-level tokenization only).",
+    )
+    parser.add_argument(
+        "--separator",
+        "-s",
+        default="\n",
+        help="Token separator in output (default: newline).",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> None:
+    """Entry point for the ``kannada-tokenize`` CLI.
+
+    Args:
+        argv: Optional argument list (defaults to ``sys.argv[1:]``).
+    """
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    # Determine input source
+    if args.text is not None:
+        text: str = args.text
+    elif not sys.stdin.isatty():
+        text = sys.stdin.read()
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    # Select tokenization strategy
+    tok_fn = tokenize_words if args.no_sandhi else tokenize
+    tokens: list[str] = tok_fn(text)
+
+    if tokens:
+        print(args.separator.join(tokens))
+
+
+if __name__ == "__main__":
+    main()

kannada_tokenizer/sandhi.py

@@ -0,0 +1,499 @@
+"""
+sandhi.py — Rule-based Kannada sandhi splitting for Information Retrieval.
+
+Sandhi (ಸಂಧಿ) is the euphonic combination that occurs at morpheme and word
+boundaries in Kannada.  This module implements *reverse* sandhi rules: given
+a combined surface form it tries to recover the original components.
+
+Design decisions
+~~~~~~~~~~~~~~~~
+* **ISO 15919 encoding only.**  The upstream transliteration module normalises
+  all input to ISO 15919 before this module is called.
+* **Zero external dependencies.**  Pure-Python, stdlib only.
+* **Longest-match / fewest-parts heuristic.**  When multiple candidate splits
+  are possible we prefer the one that produces the fewest, longest tokens.
+  This is a good proxy for "most likely intended reading" in an IR context
+  where recall matters more than philological precision.
+
+Kannada sandhi differs from Sanskrit sandhi in several important ways:
+
+1. **Lōpa Sandhi (ಲೋಪ ಸಂಧಿ)** — Vowel elision: when two vowels meet at a
+   word boundary, one (usually the first) is dropped.
+2. **Āgama Sandhi (ಆಗಮ ಸಂಧಿ)** — Insertion: a consonant ('y' or 'v') is
+   inserted between vowels to prevent hiatus.
+3. **Ādeśa Sandhi (ಆದೇಶ ಸಂಧಿ)** — Substitution: a sound at the junction is
+   replaced by another (Guṇa-like changes in tatsama words).
+4. **Consonant Sandhi** — Voicing assimilation and gemination at word
+   boundaries, common in Dravidian Kannada.
+
+The rules are represented as 4-tuples:
+
+    (combined_pattern, left_part, right_part, category)
+
+``combined_pattern`` is the string that appears at the junction in the
+surface form.  ``left_part`` is what the *end* of the first word should be
+restored to, and ``right_part`` is what the *start* of the second word
+should be restored to.  ``category`` is one of ``'lōpa'``, ``'āgama'``,
+``'ādeśa'``, or ``'consonant'``.
+
+References
+~~~~~~~~~~
+* Kittel, *A Grammar of the Kannada Language*
+* Narasimhachar, *Kannada Vyākaraṇa*
+"""
+
+from __future__ import annotations
+
+from typing import List, Tuple
+
+# ---------------------------------------------------------------------------
+# Constants — vowel / consonant inventories (ISO 15919)
+# ---------------------------------------------------------------------------
+
+VOWELS: set[str] = {
+    "a", "ā", "i", "ī", "u", "ū",
+    "e", "ē", "ai", "o", "ō", "au",
+}
+
+SHORT_VOWELS: set[str] = {"a", "i", "u", "e", "o"}
+
+LONG_VOWELS: set[str] = {"ā", "ī", "ū", "ē", "ō"}
+
+DIPHTHONGS: set[str] = {"ai", "au"}
+
+VOICED_CONSONANTS: set[str] = {
+    "g", "gh", "j", "jh", "ḍ", "ḍh", "d", "dh", "b", "bh",
+    "ṅ", "ñ", "ṇ", "n", "m",
+    "y", "r", "l", "v", "ḷ",
+    "h",
+}
+
+UNVOICED_CONSONANTS: set[str] = {
+    "k", "kh", "c", "ch", "ṭ", "ṭh", "t", "th", "p", "ph",
+    "s", "ś", "ṣ",
+}
+
+STOPS_VOICED: dict[str, str] = {
+    # unvoiced stop → voiced counterpart
+    "k": "g", "kh": "gh",
+    "c": "j", "ch": "jh",
+    "ṭ": "ḍ", "ṭh": "ḍh",
+    "t": "d", "th": "dh",
+    "p": "b", "ph": "bh",
+}
+
+NASALS_FOR_CLASS: dict[str, str] = {
+    # class stop → class nasal
+    "k": "ṅ", "kh": "ṅ", "g": "ṅ", "gh": "ṅ",
+    "c": "ñ", "ch": "ñ", "j": "ñ", "jh": "ñ",
+    "ṭ": "ṇ", "ṭh": "ṇ", "ḍ": "ṇ", "ḍh": "ṇ",
+    "t": "n", "th": "n", "d": "n", "dh": "n",
+    "p": "m", "ph": "m", "b": "m", "bh": "m",
+}
+
+# ---------------------------------------------------------------------------
+# SANDHI RULES — reverse (splitting) direction
+# ---------------------------------------------------------------------------
+#
+# Each tuple is: (combined_pattern, left_part, right_part, category)
+#
+# ``combined_pattern`` is what we *see* at the junction in the surface form.
+# ``left_part``        is the restored tail of the first word.
+# ``right_part``       is the restored head of the second word.
+# ``category``         is one of 'lōpa', 'āgama', 'ādeśa', 'consonant'.
+#
+# Rules are ordered longest-pattern-first so that a simple linear scan with
+# ``str.find`` will naturally prefer the most specific match.
+
+# ---- 1. Lōpa Sandhi (ಲೋಪ ಸಂಧಿ) — Vowel Elision ----------------------------
+#
+# When two vowels meet at a word boundary, one is dropped (usually the
+# first vowel).  Reversing this means: where we see a single vowel at a
+# potential junction, we try restoring the elided vowel.
+#
+# Common patterns:
+#   a + a → a  (first 'a' dropped)
+#   a + ā → ā
+#   a + u → u
+#   a + i → i
+#   a + e → e
+#   a + ō → ō
+#
+# General principle: short vowel before a different vowel is dropped.
+
+_LOPA_SANDHI_RULES: List[Tuple[str, str, str, str]] = []
+
+# a + a → a  (identical vowel merger — most common)
+_LOPA_SANDHI_RULES.append(("a", "a", "a", "lōpa"))
+
+# a + long/different vowel → the second vowel survives
+_LOPA_PAIRS: list[tuple[str, str]] = [
+    # (surface_vowel, restored_right_start)
+    ("ā", "ā"),
+    ("i", "i"),
+    ("ī", "ī"),
+    ("u", "u"),
+    ("ū", "ū"),
+    ("e", "e"),
+    ("ē", "ē"),
+    ("o", "o"),
+    ("ō", "ō"),
+    ("ai", "ai"),
+    ("au", "au"),
+]
+for _surface, _right in _LOPA_PAIRS:
+    # 'a' was elided before _right: reverse means surface _surface → 'a' + _right
+    _LOPA_SANDHI_RULES.append((_surface, "a", _right, "lōpa"))
+
+# i/ī elision before dissimilar vowels (less common but attested)
+for _short_v, _long_v in [("i", "ī"), ("u", "ū")]:
+    for _following in ["a", "ā", "e", "ē", "o", "ō"]:
+        _LOPA_SANDHI_RULES.append((_following, _short_v, _following, "lōpa"))
+
+
+# ---- 2. Āgama Sandhi (ಆಗಮ ಸಂಧಿ) — Insertion --------------------------------
+#
+# A consonant (usually 'y' or 'v') is inserted between vowels to avoid
+# hiatus.  Similar to Sanskrit yān sandhi.
+#
+# Patterns:
+#   i/ī + vowel → i/ī + y + vowel  →  reverse: 'y' at junction → boundary
+#   u/ū + vowel → u/ū + v + vowel  →  reverse: 'v' at junction → boundary
+#
+# We store these with an empty right_part; the split logic fills in the
+# actual vowel from the surface form.
+
+_AGAMA_SANDHI_RULES: List[Tuple[str, str, str, str]] = []
+
+# y-āgama: after i/ī
+for _src in ("i", "ī"):
+    _AGAMA_SANDHI_RULES.append(("y", _src, "", "āgama"))
+
+# v-āgama: after u/ū
+for _src in ("u", "ū"):
+    _AGAMA_SANDHI_RULES.append(("v", _src, "", "āgama"))
+
+
+# ---- 3. Ādeśa Sandhi (ಆದೇಶ ಸಂಧಿ) — Substitution ----------------------------
+#
+# A sound at the junction is replaced by another.  This covers Guṇa-like
+# changes that appear in tatsama (Sanskrit-borrowed) words in Kannada.
+#
+# Guṇa patterns (from Sanskrit influence):
+#   a/ā + i/ī → e/ē
+#   a/ā + u/ū → o/ō
+#   a/ā + ṛ   → ar
+
+_ADESA_SANDHI_RULES: List[Tuple[str, str, str, str]] = []
+
+# Guṇa: a/ā + i/ī → e/ē
+for _a in ("a", "ā"):
+    for _i in ("i", "ī"):
+        _ADESA_SANDHI_RULES.append(("e", _a, _i, "ādeśa"))
+        _ADESA_SANDHI_RULES.append(("ē", _a, _i, "ādeśa"))
+
+# Guṇa: a/ā + u/ū → o/ō
+for _a in ("a", "ā"):
+    for _u in ("u", "ū"):
+        _ADESA_SANDHI_RULES.append(("o", _a, _u, "ādeśa"))
+        _ADESA_SANDHI_RULES.append(("ō", _a, _u, "ādeśa"))
+
+# Guṇa: a/ā + ṛ → ar (in tatsama words)
+_ADESA_SANDHI_RULES.append(("ar", "a", "ṛ", "ādeśa"))
+_ADESA_SANDHI_RULES.append(("ar", "ā", "ṛ", "ādeśa"))
+
+# Vṛddhi-like: ā + i/ī → ai; ā + u/ū → au (in tatsama words)
+for _i in ("i", "ī"):
+    _ADESA_SANDHI_RULES.append(("ai", "ā", _i, "ādeśa"))
+for _u in ("u", "ū"):
+    _ADESA_SANDHI_RULES.append(("au", "ā", _u, "ādeśa"))
+
+
+# ---- 4. Consonant Sandhi ----------------------------------------------------
+#
+# 4a. Voicing assimilation — an originally unvoiced final stop becomes
+#     voiced before a voiced consonant / vowel in the next word.
+#
+# 4b. Gemination — consonant doubling at word boundaries, very common in
+#     Dravidian Kannada (e.g., compound words often geminate the initial
+#     consonant of the second member).
+#
+# 4c. Nasal assimilation — a final nasal assimilates to the class of a
+#     following stop.
+
+_CONSONANT_SANDHI_RULES: List[Tuple[str, str, str, str]] = []
+
+# 4a. Voicing assimilation — voiced stop at end could be unvoiced original
+for _unv, _voi in STOPS_VOICED.items():
+    _CONSONANT_SANDHI_RULES.append((_voi, _unv, "", "consonant"))
+
+# 4b. Gemination — doubled consonant at junction → split before second
+#     This is a very productive pattern in Kannada compounds.
+_GEMINATION_CONSONANTS: list[str] = [
+    "k", "g", "c", "j", "ṭ", "ḍ", "t", "d", "p", "b",
+    "m", "n", "ṇ", "ṅ", "ñ",
+    "y", "r", "l", "v", "ḷ",
+    "s", "ś", "ṣ", "h",
+]
+for _c in _GEMINATION_CONSONANTS:
+    # doubled consonant → first word ends with single, second word starts
+    # with single (the gemination was introduced at the boundary)
+    _CONSONANT_SANDHI_RULES.append((_c + _c, _c, _c, "consonant"))
+
+# 4c. Nasal assimilation — class nasal before a stop ← original 'm' or 'n'
+for _stop, _nasal in NASALS_FOR_CLASS.items():
+    _CONSONANT_SANDHI_RULES.append((_nasal + _stop, "m", _stop, "consonant"))
+    _CONSONANT_SANDHI_RULES.append((_nasal + _stop, "n", _stop, "consonant"))
+
+# 4d. Final 't' + sibilant combinations (in tatsama words)
+_CONSONANT_SANDHI_RULES.append(("cch", "t", "ś", "consonant"))
+_CONSONANT_SANDHI_RULES.append(("cc", "t", "c", "consonant"))
+_CONSONANT_SANDHI_RULES.append(("jj", "t", "j", "consonant"))
+
+# 4e. Final 't' → 'd' before voiced consonant (generic catch-all)
+_CONSONANT_SANDHI_RULES.append(("d", "t", "", "consonant"))
+
+
+# ---------------------------------------------------------------------------
+# Combined rule list — sorted longest pattern first for greedy matching
+# ---------------------------------------------------------------------------
+
+SANDHI_RULES: List[Tuple[str, str, str, str]] = sorted(
+    _LOPA_SANDHI_RULES
+    + _AGAMA_SANDHI_RULES
+    + _ADESA_SANDHI_RULES
+    + _CONSONANT_SANDHI_RULES,
+    key=lambda r: len(r[0]),
+    reverse=True,
+)
+"""All reverse-sandhi rules, ordered longest ``combined_pattern`` first.
+
+Each entry is ``(combined_pattern, left_part, right_part, category)`` where
+*combined_pattern* is the surface string at the junction, *left_part* is
+the restored ending of the first token, *right_part* is the restored
+beginning of the second token (empty string when context-dependent), and
+*category* is one of ``'lōpa'``, ``'āgama'``, ``'ādeśa'``, or
+``'consonant'``.
+"""
+
+
+# ---------------------------------------------------------------------------
+# Candidate generation
+# ---------------------------------------------------------------------------
+
+_SINGLE_VOWEL_CHARS: set[str] = {
+    "a", "ā", "i", "ī", "u", "ū", "e", "ē", "o", "ō",
+}
+
+_CONSONANT_CHARS: set[str] = {
+    "k", "g", "c", "j", "ṭ", "ḍ", "t", "d", "p", "b",
+    "ṅ", "ñ", "ṇ", "n", "m",
+    "y", "r", "l", "v", "ḷ",
+    "ś", "ṣ", "s", "h",
+}
+
+# Bases that form multi-char aspirated consonants with a following 'h'
+_ASPIRATE_BASES: set[str] = {
+    "k", "g", "c", "j", "ṭ", "ḍ", "t", "d", "p", "b",
+}
+
+# Categories that are vowel-type (require consonant-before-junction check)
+_VOWEL_TYPE_CATEGORIES: set[str] = {"lōpa", "āgama", "ādeśa"}
+
+
+def _generate_candidates(word: str) -> list[list[str]]:
+    """Generate all possible sandhi-split candidates for *word*.
+
+    For every position in the word where a rule's ``combined_pattern``
+    matches, we create a candidate split ``[left, right]`` by replacing the
+    combined pattern with the rule's left and right parts.
+
+    We also recurse on the *right* fragment so that chains of sandhi
+    (common in long compound words) are handled.
+
+    Returns
+    -------
+    list[list[str]]
+        Each element is a list of ISO 15919 string fragments.  If no rule
+        applies anywhere in the word the returned list is empty — the
+        caller should fall back to ``[word]``.
+    """
+    candidates: list[list[str]] = []
+
+    for combined, left_part, right_part, category in SANDHI_RULES:
+        clen = len(combined)
+        if clen == 0:
+            continue
+
+        # Scan every possible junction position.  A junction cannot be at
+        # position 0 (nothing on the left) or at position len(word) (nothing
+        # on the right).  We need at least one character on each side of the
+        # combined pattern to form two meaningful tokens.
+        for pos in range(1, len(word) - clen + 1):
+            # Check if the combined pattern sits at this position
+            if word[pos: pos + clen] != combined:
+                continue
+
+            # Context validation for vowel-type sandhi (lōpa, āgama, ādeśa):
+            # the character immediately before the junction must be a
+            # consonant.  Vowel-type sandhi occurs at the boundary between a
+            # consonant-final stem and a vowel-initial word; the consonant
+            # before the fused/inserted element is the last consonant of
+            # word-1.  Without this check, rules would spuriously match
+            # inside words.
+            if category in _VOWEL_TYPE_CATEGORIES:
+                char_before = word[pos - 1].lower()
+                if char_before not in _CONSONANT_CHARS:
+                    continue
+
+                # Guard against splitting inside ISO 15919 multi-char
+                # consonants.  If char_before is 'h' and the char before
+                # *that* is a stop, then we're inside an aspirated consonant
+                # (dh, bh, kh, etc.) and this is NOT a valid junction point.
+                if (
+                    char_before == "h"
+                    and pos >= 2
+                    and word[pos - 2].lower() in _ASPIRATE_BASES
+                ):
+                    continue
+
+                # Also need material after the junction for it to be a split
+                if not word[pos + clen:]:
+                    continue
+
+            # ----- reject trivial / degenerate splits -----
+            left_token = word[:pos] + left_part
+            right_token = right_part + word[pos + clen:]
+
+            if not left_token or not right_token:
+                continue
+
+            # Record the direct two-way split
+            candidates.append([left_token, right_token])
+
+            # Recurse on the right fragment to handle compound sandhi
+            sub_candidates = _generate_candidates(right_token)
+            for sub in sub_candidates:
+                candidates.append([left_token] + sub)
+
+    return candidates
+
+
+# ---------------------------------------------------------------------------
+# Candidate scoring
+# ---------------------------------------------------------------------------
+
+def _score_candidate(parts: list[str]) -> float:
+    """Score a candidate split — higher is better.
+
+    The scoring heuristic balances two goals:
+
+    1. **Fewer parts** — every additional split risks an incorrect break, so
+       we penalise the number of parts.
+    2. **Longer tokens** — very short (1-char) fragments are suspicious; we
+       reward candidates whose shortest token is longer.
+
+    The formula is intentionally simple so that it is fast and transparent::
+
+        score = min_token_length * 10 + avg_length * 2 - num_parts * 5
+
+    A more sophisticated scorer could consult a dictionary or n-gram model,
+    but for IR purposes this works well enough.
+
+    Parameters
+    ----------
+    parts : list[str]
+        The candidate list of tokens.
+
+    Returns
+    -------
+    float
+        A numeric score; higher values indicate better candidates.
+    """
+    if not parts:
+        return -1.0
+
+    num_parts = len(parts)
+    min_len = min(len(p) for p in parts)
+    avg_len = sum(len(p) for p in parts) / num_parts
+
+    # Prefer fewer parts (penalty) and longer minimum token (reward).
+    # The average length provides a secondary signal.
+    score = (min_len * 10.0) + (avg_len * 2.0) - (num_parts * 5.0)
+    return score
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def split_sandhi(word: str) -> list[str]:
+    """Split a single ISO 15919-encoded Kannada word at sandhi junctions.
+
+    Uses a rule-based approach: every reverse-sandhi rule is tried at every
+    position in the word, candidate splits are scored, and the highest-scoring
+    split is returned.  If no rule produces a plausible split the word is
+    returned unchanged as ``[word]``.
+
+    Parameters
+    ----------
+    word : str
+        A single token in ISO 15919 transliteration.
+
+    Returns
+    -------
+    list[str]
+        One or more ISO 15919 tokens resulting from the split.  Guaranteed
+        to be non-empty.
+
+    Examples
+    --------
+    >>> split_sandhi("rāmāyana")
+    ['rāma', 'āyana']
+    >>> split_sandhi("dēvi")        # no split needed (too short / no rule)
+    ['dēvi']
+    """
+    if not word or len(word) <= 1:
+        return [word] if word else [""]
+
+    # Short words (≤5 chars) are very unlikely to contain a real sandhi
+    # junction that splits into two meaningful tokens.
+    if len(word) <= 5:
+        return [word]
+
+    candidates = _generate_candidates(word)
+
+    if not candidates:
+        return [word]
+
+    # Filter candidates: every fragment must have at least 3 characters
+    # to avoid degenerate splits like 'dha' + 'rma' from 'dharma'.
+    _MIN_TOKEN_LEN = 3
+    viable: list[list[str]] = []
+    for candidate in candidates:
+        if any(len(p) < _MIN_TOKEN_LEN for p in candidate):
+            continue
+        viable.append(candidate)
+
+    if not viable:
+        return [word]
+
+    # Score each viable candidate and pick the best one.
+    best: list[str] = viable[0]
+    best_score = _score_candidate(viable[0])
+
+    for candidate in viable[1:]:
+        score = _score_candidate(candidate)
+        if score > best_score:
+            best_score = score
+            best = candidate
+
+    # Only accept the split if it scores reasonably well compared to
+    # keeping the word intact.  This prevents spurious splits on normal
+    # words like 'kannaḍa' where lōpa rules match but the split is
+    # nonsensical.  The 0.8 factor biases toward splitting for IR recall.
+    unsplit_score = _score_candidate([word])
+    if best_score < unsplit_score * 0.8:
+        return [word]
+
+    return best

kannada_tokenizer/tokenizer.py

@@ -0,0 +1,74 @@
+"""Main tokenizer module for Kannada text processing.
+
+Mirrors the Sanskrit tokenizer architecture, adapted for Kannada script
+(Unicode block U+0C80–U+0CFF) and ISO 15919 transliteration.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .transliterate import normalize_to_iso15919
+from .sandhi import split_sandhi
+
+
+# Punctuation pattern: Kannada-specific punctuation, Devanagari dandas,
+# common Latin punctuation, and whitespace.
+_SPLIT_PATTERN = re.compile(
+    r"[\s।॥\.\,\;\:\!\?\"\'\(\)\[\]\{\}\-\—\–]+"
+)
+
+
+def tokenize(text: str) -> list[str]:
+    """Tokenize Kannada text with transliteration normalization and sandhi splitting.
+
+    Pipeline:
+        1. Normalize input to ISO 15919.
+        2. Split on whitespace and punctuation (dandas, periods, commas, etc.).
+        3. Attempt sandhi splitting on each word.
+        4. Lowercase all tokens and filter empties.
+
+    Args:
+        text: Input Kannada text in any supported script/transliteration.
+
+    Returns:
+        A flat list of lowercase token strings.
+    """
+    if not text or not text.strip():
+        return []
+
+    normalized: str = normalize_to_iso15919(text)
+    raw_words: list[str] = _SPLIT_PATTERN.split(normalized)
+
+    tokens: list[str] = []
+    for word in raw_words:
+        word = word.strip()
+        if not word:
+            continue
+        # Attempt sandhi splitting; returns a list of sub-tokens
+        sub_tokens: list[str] = split_sandhi(word)
+        tokens.extend(sub_tokens)
+
+    # Lowercase and final empty filter
+    return [t.lower() for t in tokens if t.strip()]
+
+
+def tokenize_words(text: str) -> list[str]:
+    """Tokenize Kannada text at word boundaries only (no sandhi splitting).
+
+    Useful when sandhi analysis is not needed or when working with
+    pre-split text.
+
+    Args:
+        text: Input Kannada text in any supported script/transliteration.
+
+    Returns:
+        A flat list of lowercase word-level tokens.
+    """
+    if not text or not text.strip():
+        return []
+
+    normalized: str = normalize_to_iso15919(text)
+    raw_words: list[str] = _SPLIT_PATTERN.split(normalized)
+
+    return [w.lower() for w in raw_words if w.strip()]

kannada_tokenizer/transliterate.py

@@ -0,0 +1,599 @@
+"""
+transliterate.py — Bidirectional Kannada ↔ ISO 15919 transliteration.
+
+This module provides pure-Python, zero-dependency conversion between
+Kannada script and ISO 15919 romanization (IAST extended for Dravidian
+languages).
+
+Public API
+----------
+- kannada_to_iso15919(text)     — Convert Kannada text to ISO 15919.
+- iso15919_to_kannada(text)     — Convert ISO 15919 text to Kannada.
+- is_kannada(text)              — Detect whether a string is primarily Kannada.
+- normalize_to_iso15919(text)   — Auto-detect script and return ISO 15919.
+
+Design notes
+------------
+* All mapping data lives in plain dicts/tuples at module level so tables can be
+  maintained, extended, or serialised without touching logic.
+* Unicode NFC normalisation is applied at every public entry-point.
+* Mixed-script text (e.g. Kannada mixed with ASCII punctuation) is handled
+  gracefully: characters outside the relevant script are passed through as-is.
+* Kannada distinguishes short e/o (ಎ/ಒ) from long ē/ō (ಏ/ಓ), unlike
+  Devanagari. The ISO 15919 standard uses macrons (ē, ō) for the long forms.
+* Two-part vowel signs (e.g. ೊ = \u0CC6 + \u0CBE) are handled as single
+  precomposed characters after NFC normalization.
+"""
+
+from __future__ import annotations
+
+import unicodedata
+from typing import Dict, List, Tuple
+
+# ──────────────────────────────────────────────────────────────────────
+# §1  Constants — Unicode code-point ranges
+# ──────────────────────────────────────────────────────────────────────
+
+# Kannada block: U+0C80 – U+0CFF
+_KANNADA_START = 0x0C80
+_KANNADA_END = 0x0CFF
+
+# Virama (halant) — suppresses the inherent 'a' vowel
+VIRAMA = "\u0CCD"  # ್
+
+# Nukta — modifies a consonant to represent a borrowed sound
+NUKTA = "\u0CBC"  # ಼
+
+# Chandrabindu, Anusvara, Visarga
+CHANDRABINDU = "\u0C81"  # ಁ
+ANUSVARA = "\u0C82"  # ಂ
+VISARGA = "\u0C83"  # ಃ
+
+# ──────────────────────────────────────────────────────────────────────
+# §2  Mapping tables — Kannada → ISO 15919
+# ──────────────────────────────────────────────────────────────────────
+#
+# Tables are split into logical groups for clarity. Every group is a
+# plain dict mapping a *single* Kannada character (or short sequence
+# in the case of nukta consonants) to its ISO 15919 equivalent.
+#
+# For consonants the ISO 15919 value does NOT include the inherent 'a';
+# that is inserted by the transliteration engine when no mātrā or
+# virama follows.
+#
+# NOTE: Kannada has a richer vowel inventory than Devanagari — it
+# distinguishes short e/o from long ē/ō.  ISO 15919 uses macrons
+# for the long forms.
+
+# ----- 2a. Independent vowels (svara) -----
+# These appear at the start of a word or after another vowel.
+VOWELS_INDEPENDENT: Dict[str, str] = {
+    "ಅ": "a",    # U+0C85
+    "ಆ": "ā",    # U+0C86
+    "ಇ": "i",    # U+0C87
+    "ಈ": "ī",    # U+0C88
+    "ಉ": "u",    # U+0C89
+    "ಊ": "ū",    # U+0C8A
+    "ಋ": "ṛ",    # U+0C8B
+    "ೠ": "ṝ",    # U+0CE0
+    "ಌ": "ḷ",    # U+0C8C
+    "ೡ": "ḹ",    # U+0CE1
+    "ಎ": "e",    # U+0C8E — short e (Dravidian distinction!)
+    "ಏ": "ē",    # U+0C8F — long ē
+    "ಐ": "ai",   # U+0C90
+    "ಒ": "o",    # U+0C92 — short o (Dravidian distinction!)
+    "ಓ": "ō",    # U+0C93 — long ō
+    "ಔ": "au",   # U+0C94
+}
+
+# ----- 2b. Dependent vowel signs (mātrā) -----
+# These attach to a preceding consonant and replace the inherent 'a'.
+# Note: there is no mātrā for 'a' because 'a' is the inherent vowel.
+#
+# Some Kannada vowel signs are "two-part":
+#   ೊ (U+0CCA) = ೆ (U+0CC6) + ಾ (U+0CBE)
+#   ೋ (U+0CCB) = ೇ (U+0CC7) + ಾ (U+0CBE)
+#   ೌ (U+0CCC) = ೆ (U+0CC6) + ೕ (U+0CD5)
+# NFC normalization collapses these into the precomposed forms listed below.
+VOWELS_DEPENDENT: Dict[str, str] = {
+    "\u0CBE": "ā",   # ಾ
+    "\u0CBF": "i",   # ಿ
+    "\u0CC0": "ī",   # ೀ
+    "\u0CC1": "u",   # ು
+    "\u0CC2": "ū",   # ೂ
+    "\u0CC3": "ṛ",   # ೃ
+    "\u0CC4": "ṝ",   # ೄ
+    "\u0CE2": "ḷ",   # ೢ
+    "\u0CE3": "ḹ",   # ೣ
+    "\u0CC6": "e",   # ೆ — short e
+    "\u0CC7": "ē",   # ೇ — long ē
+    "\u0CC8": "ai",  # ೈ
+    "\u0CCA": "o",   # ೊ — short o (precomposed two-part sign)
+    "\u0CCB": "ō",   # ೋ — long ō (precomposed two-part sign)
+    "\u0CCC": "au",  # ೌ
+}
+
+# ----- 2c. Consonants (vyañjana) -----
+# Arranged by the traditional varga (class) system, plus Dravidian-specific
+# consonants at the end.
+CONSONANTS: Dict[str, str] = {
+    # -- Velar (kaṇṭhya) --
+    "ಕ": "k",     # U+0C95
+    "ಖ": "kh",    # U+0C96
+    "ಗ": "g",     # U+0C97
+    "ಘ": "gh",    # U+0C98
+    "ಙ": "ṅ",     # U+0C99
+    # -- Palatal (tālavya) --
+    "ಚ": "c",     # U+0C9A
+    "ಛ": "ch",    # U+0C9B
+    "ಜ": "j",     # U+0C9C
+    "ಝ": "jh",    # U+0C9D
+    "ಞ": "ñ",     # U+0C9E
+    # -- Retroflex (mūrdhanya) --
+    "ಟ": "ṭ",     # U+0C9F
+    "ಠ": "ṭh",    # U+0CA0
+    "ಡ": "ḍ",     # U+0CA1
+    "ಢ": "ḍh",    # U+0CA2
+    "ಣ": "ṇ",     # U+0CA3
+    # -- Dental (dantya) --
+    "ತ": "t",     # U+0CA4
+    "ಥ": "th",    # U+0CA5
+    "ದ": "d",     # U+0CA6
+    "ಧ": "dh",    # U+0CA7
+    "ನ": "n",     # U+0CA8
+    # -- Labial (oṣṭhya) --
+    "ಪ": "p",     # U+0CAA
+    "ಫ": "ph",    # U+0CAB
+    "ಬ": "b",     # U+0CAC
+    "ಭ": "bh",    # U+0CAD
+    "ಮ": "m",     # U+0CAE
+    # -- Semi-vowels (antaḥstha) --
+    "ಯ": "y",     # U+0CAF
+    "ರ": "r",     # U+0CB0
+    "ಲ": "l",     # U+0CB2
+    "ವ": "v",     # U+0CB5
+    # -- Sibilants (ūṣman) --
+    "ಶ": "ś",     # U+0CB6
+    "ಷ": "ṣ",     # U+0CB7
+    "ಸ": "s",     # U+0CB8
+    # -- Glottal --
+    "ಹ": "h",     # U+0CB9
+    # -- Dravidian-specific --
+    "ಳ": "ḻ",     # U+0CB3 — retroflex lateral (unique to Dravidian!)
+    "ೞ": "ẕ",     # U+0CDE — retroflex approximant (archaic Kannada)
+}
+
+# ----- 2d. Nukta consonants (borrowed sounds) -----
+# Each is the base consonant + nukta (U+0CBC).
+# Less commonly used in Kannada than in Devanagari, but supported for
+# loanword representation.
+NUKTA_CONSONANTS: Dict[str, str] = {
+    "ಕ಼": "q",     # ಕ + ಼
+    "ಖ಼": "x",     # ಖ + ಼
+    "ಗ಼": "ġ",     # ಗ + ಼
+    "ಜ಼": "z",     # ಜ + ಼
+    "ಫ಼": "f",     # ಫ + ಼
+}
+
+# ----- 2e. Modifiers & signs -----
+MODIFIERS: Dict[str, str] = {
+    ANUSVARA:     "ṃ",
+    VISARGA:      "ḥ",
+    CHANDRABINDU: "m̐",   # nasalisation mark
+}
+
+# ----- 2f. Kannada digits -----
+DIGITS: Dict[str, str] = {
+    "೦": "0",   # U+0CE6
+    "೧": "1",   # U+0CE7
+    "೨": "2",   # U+0CE8
+    "೩": "3",   # U+0CE9
+    "೪": "4",   # U+0CEA
+    "೫": "5",   # U+0CEB
+    "೬": "6",   # U+0CEC
+    "೭": "7",   # U+0CED
+    "೮": "8",   # U+0CEE
+    "೯": "9",   # U+0CEF
+}
+
+# ----- 2g. Punctuation -----
+# Danda and double danda are shared across Indic scripts (Devanagari block).
+PUNCTUATION: Dict[str, str] = {
+    "\u0964": ".",   # । danda  → full stop
+    "\u0965": ".",   # ॥ double danda → full stop
+}
+
+# ──────────────────────────────────────────────────────────────────────
+# §3  Reverse mapping tables — ISO 15919 → Kannada
+# ──────────────────────────────────────────────────────────────────────
+#
+# Built programmatically from the forward tables where possible, with
+# hand-tuned additions for multi-character ISO 15919 sequences that must
+# be matched longest-first to avoid ambiguity (e.g. "kh" before "k").
+
+def _invert(d: Dict[str, str]) -> Dict[str, str]:
+    """Return {v: k for k, v in d.items()}, preserving first occurrence."""
+    inv: Dict[str, str] = {}
+    for k, v in d.items():
+        if v not in inv:
+            inv[v] = k
+    return inv
+
+
+# Independent vowels: ISO 15919 → Kannada independent vowel
+_ISO15919_TO_VOWEL_INDEPENDENT: Dict[str, str] = _invert(VOWELS_INDEPENDENT)
+
+# Dependent vowels: ISO 15919 → Kannada mātrā
+# 'a' maps to empty string because the inherent vowel needs no mātrā.
+_ISO15919_TO_VOWEL_DEPENDENT: Dict[str, str] = _invert(VOWELS_DEPENDENT)
+_ISO15919_TO_VOWEL_DEPENDENT["a"] = ""  # inherent vowel — no visible mātrā
+
+# Consonants: ISO 15919 → Kannada base consonant
+_ISO15919_TO_CONSONANT: Dict[str, str] = _invert(CONSONANTS)
+
+# Nukta consonants
+_ISO15919_TO_NUKTA: Dict[str, str] = _invert(NUKTA_CONSONANTS)
+
+# Modifiers
+_ISO15919_TO_MODIFIER: Dict[str, str] = _invert(MODIFIERS)
+
+# Digits
+_ISO15919_TO_DIGIT: Dict[str, str] = _invert(DIGITS)
+
+# All ISO 15919 vowel strings (used for lookahead when deciding inherent 'a').
+_ALL_ISO15919_VOWELS: set = set(VOWELS_INDEPENDENT.values())
+
+# Set of all Kannada dependent-vowel (mātrā) code-points.
+_MATRA_CODEPOINTS: set = set(VOWELS_DEPENDENT.keys())
+
+# Set of all Kannada independent-vowel characters.
+_INDEPENDENT_VOWEL_CODEPOINTS: set = set(VOWELS_INDEPENDENT.keys())
+
+# Set of all Kannada consonant characters (without nukta).
+_CONSONANT_CODEPOINTS: set = set(CONSONANTS.keys())
+
+# ──────────────────────────────────────────────────────────────────────
+# §4  Helper — sorted ISO 15919 tokens for longest-match parsing
+# ──────────────────────────────────────────────────────────────────────
+
+def _build_sorted_iso15919_tokens() -> List[Tuple[str, str, str]]:
+    """
+    Return a list of (iso15919_token, kannada, category) tuples sorted by
+    descending token length so that the parser always tries the longest
+    match first (e.g. "kh" before "k", "ai" before "a").
+
+    Categories: 'consonant', 'vowel', 'modifier', 'digit', 'nukta'.
+    """
+    tokens: List[Tuple[str, str, str]] = []
+
+    for iso, kan in _ISO15919_TO_CONSONANT.items():
+        tokens.append((iso, kan, "consonant"))
+    for iso, kan in _ISO15919_TO_NUKTA.items():
+        tokens.append((iso, kan, "nukta"))
+    for iso, kan in _ISO15919_TO_VOWEL_INDEPENDENT.items():
+        tokens.append((iso, kan, "vowel"))
+    for iso, kan in _ISO15919_TO_MODIFIER.items():
+        tokens.append((iso, kan, "modifier"))
+    for iso, kan in _ISO15919_TO_DIGIT.items():
+        tokens.append((iso, kan, "digit"))
+
+    # Sort descending by length so longest tokens are tried first
+    tokens.sort(key=lambda t: len(t[0]), reverse=True)
+    return tokens
+
+
+_ISO15919_TOKENS: List[Tuple[str, str, str]] = _build_sorted_iso15919_tokens()
+
+# ──────────────────────────────────────────────────────────────────────
+# §5  Core: Kannada → ISO 15919
+# ──────────────────────────────────────────────────────────────────────
+
+def _is_kannada_char(ch: str) -> bool:
+    """Return True if *ch* falls within the Kannada Unicode block."""
+    cp = ord(ch)
+    return _KANNADA_START <= cp <= _KANNADA_END
+
+
+def kannada_to_iso15919(text: str) -> str:
+    """
+    Convert a Kannada string to ISO 15919.
+
+    The algorithm walks the string character-by-character, maintaining a
+    flag ``pending_a`` that tracks whether the inherent vowel 'a' should
+    be emitted after a consonant.  The inherent vowel is suppressed when
+    a virama or a dependent vowel (mātrā) follows.
+
+    Non-Kannada characters (Latin letters, ASCII punctuation, …) are
+    passed through unchanged, which makes the function safe for mixed-
+    script input.
+
+    Parameters
+    ----------
+    text : str
+        Input string, possibly containing Kannada characters.
+
+    Returns
+    -------
+    str
+        ISO 15919 transliteration of *text*.
+    """
+    # Always work with NFC-normalised text so that two-part vowel signs
+    # (e.g. ೊ = ೆ + ಾ) are collapsed into their precomposed forms,
+    # and nukta sequences (base + U+0CBC) are in canonical form.
+    text = unicodedata.normalize("NFC", text)
+
+    out: List[str] = []
+    i = 0
+    n = len(text)
+    pending_a = False  # True when we've just emitted a consonant and may need 'a'
+
+    while i < n:
+        ch = text[i]
+
+        # ── Nukta consonants (two-char sequences: base + nukta) ──
+        # Check BEFORE single consonants so "ಕ಼" is matched as a unit.
+        if i + 1 < n and text[i + 1] == NUKTA:
+            pair = text[i : i + 2]
+            if pair in NUKTA_CONSONANTS:
+                # Flush any pending inherent 'a' from a previous consonant
+                if pending_a:
+                    out.append("a")
+                    pending_a = False
+                out.append(NUKTA_CONSONANTS[pair])
+                pending_a = True
+                i += 2
+                continue
+
+        # ── Virama (halant) — kills the inherent vowel ──
+        if ch == VIRAMA:
+            # Cancel the pending inherent 'a'
+            pending_a = False
+            i += 1
+            continue
+
+        # ── Dependent vowels (mātrā) — replace inherent 'a' ──
+        if ch in VOWELS_DEPENDENT:
+            # A mātrā always cancels the inherent 'a' and provides its
+            # own vowel sound.
+            pending_a = False
+            out.append(VOWELS_DEPENDENT[ch])
+            i += 1
+            continue
+
+        # ── Consonants ──
+        if ch in CONSONANTS:
+            if pending_a:
+                out.append("a")
+            out.append(CONSONANTS[ch])
+            pending_a = True
+            i += 1
+            continue
+
+        # ── Independent vowels ──
+        if ch in VOWELS_INDEPENDENT:
+            if pending_a:
+                out.append("a")
+                pending_a = False
+            out.append(VOWELS_INDEPENDENT[ch])
+            i += 1
+            continue
+
+        # ── Modifiers (anusvara, visarga, chandrabindu) ──
+        if ch in MODIFIERS:
+            # Modifiers attach to the preceding syllable; flush 'a' first.
+            if pending_a:
+                out.append("a")
+                pending_a = False
+            out.append(MODIFIERS[ch])
+            i += 1
+            continue
+
+        # ── Digits ──
+        if ch in DIGITS:
+            if pending_a:
+                out.append("a")
+                pending_a = False
+            out.append(DIGITS[ch])
+            i += 1
+            continue
+
+        # ── Punctuation ──
+        if ch in PUNCTUATION:
+            if pending_a:
+                out.append("a")
+                pending_a = False
+            out.append(PUNCTUATION[ch])
+            i += 1
+            continue
+
+        # ── Nukta by itself (shouldn't happen in well-formed text) ──
+        if ch == NUKTA:
+            i += 1
+            continue
+
+        # ── Anything else (Latin, punctuation, whitespace …) ──
+        if pending_a:
+            out.append("a")
+            pending_a = False
+        out.append(ch)
+        i += 1
+
+    # If the string ends with a consonant that still has a pending 'a'
+    if pending_a:
+        out.append("a")
+
+    return "".join(out)
+
+
+# ──────────────────────────────────────────────────────────────────────
+# §6  Core: ISO 15919 → Kannada
+# ──────────────────────────────────────────────────────────────────────
+
+def _is_iso15919_vowel_at(text: str, pos: int) -> Tuple[bool, str, int]:
+    """
+    Check whether an ISO 15919 vowel token starts at *pos* in *text*.
+
+    Returns (matched, iso15919_vowel_string, length) on success,
+    or (False, '', 0) otherwise.
+
+    Uses longest-match semantics ('ai' before 'a', 'au' before 'a', etc.).
+    """
+    # Check two-char vowels first, then one-char.
+    for length in (2, 1):
+        candidate = text[pos : pos + length].lower()
+        if candidate in _ISO15919_TO_VOWEL_INDEPENDENT:
+            return True, candidate, length
+    return False, "", 0
+
+
+def iso15919_to_kannada(text: str) -> str:
+    """
+    Convert an ISO 15919 string to Kannada.
+
+    The parser scans *text* from left to right using longest-match
+    tokenisation.  It tracks whether the current context is "after a
+    consonant" so it can choose between independent vowels and mātrā
+    forms, and inserts virama between consecutive consonants.
+
+    Parameters
+    ----------
+    text : str
+        Input string in ISO 15919 (or mixed ISO 15919 / other scripts).
+
+    Returns
+    -------
+    str
+        Kannada transliteration of *text*.
+    """
+    text = unicodedata.normalize("NFC", text)
+
+    out: List[str] = []
+    i = 0
+    n = len(text)
+    after_consonant = False  # True when the last emitted char was a consonant
+
+    while i < n:
+        matched = False
+
+        # Try every token (longest first) against the current position.
+        for iso_tok, kan, category in _ISO15919_TOKENS:
+            tok_len = len(iso_tok)
+            # Case-insensitive comparison for the ISO 15919 token
+            if text[i : i + tok_len].lower() == iso_tok.lower():
+                if category in ("consonant", "nukta"):
+                    if after_consonant:
+                        # Previous consonant needs virama before this one
+                        out.append(VIRAMA)
+                    out.append(kan)
+                    after_consonant = True
+                elif category == "vowel":
+                    if after_consonant:
+                        # Use the dependent (mātrā) form and suppress the
+                        # inherent 'a'.
+                        matra = _ISO15919_TO_VOWEL_DEPENDENT.get(iso_tok.lower(), "")
+                        out.append(matra)  # "" for 'a' → no visible mātrā
+                        after_consonant = False
+                    else:
+                        # Word-initial or after another vowel → independent form
+                        out.append(kan)
+                elif category == "modifier":
+                    # Modifiers attach after the syllable
+                    if after_consonant:
+                        # Implicit 'a' is assumed before the modifier
+                        after_consonant = False
+                    out.append(kan)
+                elif category == "digit":
+                    if after_consonant:
+                        after_consonant = False
+                    out.append(kan)
+
+                i += tok_len
+                matched = True
+                break
+
+        if not matched:
+            # Not an ISO 15919 token — pass through (spaces, ASCII punctuation, …)
+            if after_consonant:
+                after_consonant = False
+            out.append(text[i])
+            i += 1
+
+    # If the string ends mid-consonant, we must add a virama to suppress
+    # the inherent 'a'.  ISO 15919 is explicit about final vowels, so a bare
+    # consonant at end-of-string means the halant form.
+    if after_consonant:
+        out.append(VIRAMA)
+
+    return "".join(out)
+
+
+# ──────────────────────────────────────────────────────────────────────
+# §7  Detection: is_kannada
+# ──────────────────────────────────────────────────────────────────────
+
+def is_kannada(text: str, threshold: float = 0.5) -> bool:
+    """
+    Return ``True`` if *text* is primarily written in Kannada script.
+
+    The heuristic counts how many *non-whitespace* characters fall inside
+    the Kannada Unicode block and returns ``True`` when the ratio meets
+    or exceeds *threshold* (default 50 %).
+
+    Parameters
+    ----------
+    text : str
+        The text to inspect.
+    threshold : float
+        Minimum fraction of Kannada characters (among scriptable
+        characters) to consider the text "Kannada".  Defaults to 0.5.
+
+    Returns
+    -------
+    bool
+    """
+    if not text:
+        return False
+
+    text = unicodedata.normalize("NFC", text)
+
+    total = 0
+    kannada_count = 0
+    for ch in text:
+        # Skip whitespace for the ratio
+        if ch.isspace():
+            continue
+        total += 1
+        if _is_kannada_char(ch):
+            kannada_count += 1
+
+    if total == 0:
+        return False
+    return (kannada_count / total) >= threshold
+
+
+# ──────────────────────────────────────────────────────────────────────
+# §8  Convenience: normalize_to_iso15919
+# ──────────────────────────────────────────────────────────────────────
+
+def normalize_to_iso15919(text: str) -> str:
+    """
+    Auto-detect the script of *text* and return its ISO 15919 representation.
+
+    - If *text* is detected as Kannada, it is transliterated to ISO 15919.
+    - Otherwise it is returned as-is (assumed to already be ISO 15919 or some
+      other Latin-based notation).
+
+    Parameters
+    ----------
+    text : str
+        Input string in Kannada or ISO 15919.
+
+    Returns
+    -------
+    str
+        ISO 15919 representation of *text*.
+    """
+    text = unicodedata.normalize("NFC", text)
+    if is_kannada(text):
+        return kannada_to_iso15919(text)
+    return text

kannada_tokenizer-0.1.0.dist-info/METADATA

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: kannada-tokenizer
+Version: 0.1.0
+Summary: Kannada tokenizer with sandhi splitting for Information Retrieval.
+Author: Hemanth HM
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# kannada-tokenizer
+
+Tokenize Kannada text with sandhi splitting for Information Retrieval.
+
+```bash
+pip install kannada-tokenizer
+```
+
+## Quick start
+
+```python
+from kannada_tokenizer import tokenize
+
+tokenize("ಕನ್ನಡ ಧರ್ಮ")
+# ['kannaḍa', 'dharma']
+
+tokenize("vidyālaya")
+# ['vidyālaya']
+
+tokenize("mahōtsava")
+# ['maha', 'ōtsava']
+```
+
+`tokenize()` normalizes to ISO 15919, splits on whitespace and punctuation, then applies reverse sandhi rules. Accepts both Kannada script and romanized input.
+
+## Sandhi splitting
+
+```python
+from kannada_tokenizer.sandhi import split_sandhi
+
+split_sandhi("rāmāyana")   # lōpa sandhi: ā → a + ā
+# ['rāma', 'āyana']
+
+split_sandhi("kannaḍa")    # no junction found
+# ['kannaḍa']
+```
+
+Rule-based engine covering lōpa sandhi (vowel elision), āgama sandhi (y/v insertion), ādeśa sandhi (guṇa-like substitution), and consonant sandhi (voicing, gemination, nasals). Uses longest-match heuristic when splits are ambiguous.
+
+## Transliteration
+
+```python
+from kannada_tokenizer.transliterate import (
+    kannada_to_iso15919,
+    iso15919_to_kannada,
+    is_kannada,
+)
+
+kannada_to_iso15919("ಬೆಂಗಳೂರು")
+# 'beṃgaḻūru'
+
+iso15919_to_kannada("kannaḍa")
+# 'ಕನ್ನಡ'
+
+is_kannada("ಕನ್ನಡ")
+# True
+```
+
+Handles Kannada's short/long e/o distinction (ಎ→e vs ಏ→ē) and the retroflex lateral ಳ→ḻ unique to Dravidian.
+
+## Word-level tokenization
+
+```python
+from kannada_tokenizer.tokenizer import tokenize_words
+
+tokenize_words("vidyālaya namaḥ")
+# ['vidyālaya', 'namaḥ']
+```
+
+`tokenize_words()` splits on whitespace and punctuation only — no sandhi splitting.
+
+## CLI
+
+```bash
+kannada-tokenize "ಕನ್ನಡ ಧರ್ಮ"
+# kannaḍa
+# dharma
+
+echo "vidyālaya" | kannada-tokenize
+# vidyālaya
+
+kannada-tokenize --no-sandhi "mahōtsava"
+# mahōtsava
+
+kannada-tokenize -s " " "dharma yōga"
+# dharma yōga
+```
+
+- `--no-sandhi` — word-level only, skip sandhi splitting
+- `--separator SEP` — output separator (default: newline)
+
+## License
+
+MIT © [Hemanth.HM](https://h3manth.com)

kannada_tokenizer-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+kannada_tokenizer/__init__.py,sha256=b60TojvmvfbcIKpAgow0jn_qHPSS6RaDzdta5yxsHzs,169
+kannada_tokenizer/cli.py,sha256=Td9zrG3z8GWSJRUxH6b29duEQQ22QhHjiE6bPVWwl7Y,1630
+kannada_tokenizer/sandhi.py,sha256=8_uq9aZpKy8BqFnitEazjtnZgxf4yZOfEmMQ6Tw5Xek,18292
+kannada_tokenizer/tokenizer.py,sha256=grjTDyVzd_nM668lB_GzN-y3Nr99gBgjCfnn3nqIUEE,2173
+kannada_tokenizer/transliterate.py,sha256=8Himvzbnr9BXjDzT2FVgOKPvlElA7ZhXMLDCRLGhe-U,22702
+kannada_tokenizer-0.1.0.dist-info/METADATA,sha256=qOq_HJ4eKOIy5LEyNTQc8U50ZEuJQXTxj0drdha-5pE,2449
+kannada_tokenizer-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+kannada_tokenizer-0.1.0.dist-info/entry_points.txt,sha256=DyNcEFJI9f5BeV3gG0l1H4VB7_4ujcpDbvx9OwphPHY,64
+kannada_tokenizer-0.1.0.dist-info/top_level.txt,sha256=w0MkiLbHMVQDYf32CrM0hZSLgI1fjabI2BXAh_u_Bwo,18
+kannada_tokenizer-0.1.0.dist-info/RECORD,,

kannada_tokenizer-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

kannada_tokenizer-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+kannada-tokenize = kannada_tokenizer.cli:main

kannada_tokenizer-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+kannada_tokenizer