indic-itn 0.1.0__py3-none-any.whl

indic_itn/__init__.py

@@ -0,0 +1,7 @@
+"""Inverse Text Normalization (ITN) for Indic languages."""
+
+from indic_itn.main import HindiITN
+from indic_itn.tokenizer import HindiTokenizer
+
+__version__ = "0.1.0"
+__all__ = ["HindiITN", "HindiTokenizer"]

indic_itn/base.py

@@ -0,0 +1,105 @@
+"""Core domain classes and base interfaces for the ITN pipeline."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Token:
+    """A single token within the input text.
+
+    Attributes:
+        text: The text content of the token.
+        start_idx: The starting character index (inclusive) in the source text.
+        end_idx: The ending character index (exclusive) in the source text.
+    """
+
+    text: str
+    start_idx: int
+    end_idx: int
+
+    @property
+    def value(self) -> str:
+        """Alias for text."""
+        return self.text
+
+    @property
+    def start_offset(self) -> int:
+        """Alias for start_idx."""
+        return self.start_idx
+
+    @property
+    def end_offset(self) -> int:
+        """Alias for end_idx."""
+        return self.end_idx
+
+
+@dataclass(frozen=True)
+class Entity:
+    """A detected entity that requires normalization.
+
+    Attributes:
+        text: The raw spoken text representing the entity.
+        entity_type: The type of entity (e.g., "number", "date", "currency").
+        start_idx: The starting character index (inclusive) in the source text.
+        end_idx: The ending character index (exclusive) in the source text.
+        metadata: Optional dictionary containing additional structured details
+            about the detected entity.
+    """
+
+    text: str
+    entity_type: str
+    start_idx: int
+    end_idx: int
+    metadata: dict[str, Any] = field(default_factory=dict, hash=False, compare=False)
+
+
+class BaseTokenizer(ABC):
+    """Abstract base class for all tokenizers in the ITN pipeline."""
+
+    @abstractmethod
+    def tokenize(self, text: str) -> list[Token]:
+        """Split input text into a list of Token objects.
+
+        Args:
+            text: The raw input text.
+
+        Returns:
+            A list of Token objects with their corresponding indices.
+        """
+        pass
+
+
+class BaseEntityDetector(ABC):
+    """Abstract base class for detecting normalizeable entities in tokenized text."""
+
+    @abstractmethod
+    def detect(self, tokens: list[Token]) -> list[Entity]:
+        """Detect entities from the list of tokens.
+
+        Args:
+            tokens: A list of input Token objects.
+
+        Returns:
+            A list of detected Entity objects.
+        """
+        pass
+
+
+class BaseRewriter(ABC):
+    """Abstract base class for rewriting the input text with normalized values."""
+
+    @abstractmethod
+    def rewrite(self, text: str, entity_rewrites: dict[Entity, str]) -> str:
+        """Replace raw entities in the input text with their normalized string values.
+
+        Args:
+            text: The original input text.
+            entity_rewrites: A dictionary mapping each detected Entity to its
+                normalized string representation.
+
+        Returns:
+            The final normalized text.
+        """
+        pass

indic_itn/detector.py

@@ -0,0 +1,447 @@
+"""Entity detector implementation for Hindi ITN."""
+
+import re
+
+from indic_itn.base import BaseEntityDetector, Entity, Token
+from indic_itn.loader import load_hindi_resource
+
+
+class HindiEntityDetector(BaseEntityDetector):
+    """Devanagari-aware entity detector for Hindi ITN.
+
+    Identifies candidate spans for numbers, phone numbers, OTP, currency,
+    dates, and time from a list of tokens.
+    """
+
+    def __init__(self) -> None:
+        """Initializes HindiEntityDetector by loading language resources."""
+        # Load resources
+        numbers_res = load_hindi_resource("numbers")
+        temporal_res = load_hindi_resource("temporal")
+        ordinals_res = load_hindi_resource("ordinals")
+
+        # Classify vocabulary sets
+        self._digits = set(numbers_res["digits"].keys())
+        self._numbers = (
+            set(numbers_res["numbers"].keys())
+            | set(ordinals_res["ordinals"].keys())
+            | set(numbers_res["tens"].keys())
+            | set(numbers_res["hundreds"].keys())
+            | set(numbers_res["multipliers"].keys())
+            | set(numbers_res["lakh"].keys())
+            | set(numbers_res["crore"].keys())
+        )
+
+        self._months = set(temporal_res["months"].keys())
+        self._weekdays = set(temporal_res["weekdays"].keys())
+
+        # Define keyword sets
+        self._currency_words = {
+            "रुपया",
+            "रुपये",
+            "रुपए",
+            "रु",
+            "पैसे",
+            "पैसा",
+            "रू",
+            "rupees",
+            "rupee",
+            "rs",
+            "paise",
+            "paisa",
+        }
+        self._time_words = {
+            "बजे",
+            "बजकर",
+            "मिनट",
+            "सेकंड",
+            "सेकण्ड",
+            "घंटे",
+            "घण्टे",
+            "घंटा",
+            "घण्टा",
+            "सुबह",
+            "शाम",
+            "दोपहर",
+            "रात",
+            "am",
+            "pm",
+        }
+        self._time_modifiers = {
+            "साढ़े",
+            "साढ़े",
+            "सवा",
+            "पौने",
+            "डेढ़",
+            "ढाई",
+            "डेढ़",
+        }
+
+        self._decimal_indicators = {
+            "दशमलव",
+            "दशमलब",
+            "पॉइंट",
+            "पॉइन्ट",
+            "point",
+            "decimal",
+        }
+
+        self._percentage_indicators = {
+            "प्रतिशत",
+            "फीसदी",
+            "फीसदि",
+            "percent",
+            "percentage",
+        }
+
+        # Numeric regex patterns
+        self._numeric_date_pattern = re.compile(r"^\d{1,4}[-/.]\d{1,2}[-/.]\d{2,4}$")
+        self._numeric_time_pattern = re.compile(
+            r"^\d{1,2}:\d{2}(?::\d{2})?\s*(?:am|pm|AM|PM)?$"
+        )
+
+    def _get_period_len(self, tokens: list[Token], idx: int) -> int:
+        """Get the length of the period indicator starting at idx.
+
+        Returns length in tokens (0 if not a period).
+        """
+        if idx >= len(tokens):
+            return 0
+        t1 = tokens[idx].text.lower()
+        if t1 in {"am", "pm", "पीएम", "एएम"}:
+            return 1
+        if idx + 1 < len(tokens):
+            t2 = tokens[idx + 1].text.lower()
+            if (t1 == "पी" and t2 == "एम") or (t1 == "ए" and t2 == "एम"):
+                return 2
+        if idx + 3 < len(tokens):
+            t2 = tokens[idx + 1].text
+            t3 = tokens[idx + 2].text.lower()
+            t4 = tokens[idx + 3].text
+            if (t1 == "पी" and t2 == "." and t3 == "एम" and t4 == ".") or (
+                t1 == "ए" and t2 == "." and t3 == "एम" and t4 == "."
+            ):
+                return 4
+        return 0
+
+    def _is_num(self, token: Token) -> bool:
+        """Check if a token represents a digit or a number word.
+
+        Args:
+            token: The Token object.
+
+        Returns:
+            True if the token is a number/digit word or numeric digits string.
+        """
+        text = token.text
+        return (
+            text.isdigit()
+            or text in self._numbers
+            or text in self._digits
+            or text.replace(".", "", 1).isdigit()  # handles decimals
+        )
+
+    def _get_num_sequence_len(self, tokens: list[Token], start_idx: int) -> int:
+        """Get the length of a contiguous sequence of number tokens.
+
+        Args:
+            tokens: The list of tokens.
+            start_idx: The index to start scanning.
+
+        Returns:
+            The number of consecutive number tokens.
+        """
+        length = 0
+        while start_idx + length < len(tokens):
+            if self._is_num(tokens[start_idx + length]):
+                length += 1
+            else:
+                break
+        return length
+
+    def detect(self, tokens: list[Token]) -> list[Entity]:
+        """Detect entities from the list of tokens.
+
+        Args:
+            tokens: A list of input Token objects.
+
+        Returns:
+            A list of detected Entity objects.
+        """
+        entities: list[Entity] = []
+        n = len(tokens)
+        i = 0
+
+        # Helper to construct an Entity from a slice of tokens
+        def make_entity(start_pos: int, end_pos: int, entity_type: str) -> Entity:
+            span_tokens = tokens[start_pos:end_pos]
+            start_offset = tokens[start_pos].start_idx
+            end_offset = tokens[end_pos - 1].end_idx
+
+            # Reconstruct the original substring using character bounds
+            parts = []
+            for k, t in enumerate(span_tokens):
+                parts.append(t.text)
+                if k < len(span_tokens) - 1:
+                    gap = span_tokens[k + 1].start_idx - t.end_idx
+                    if gap > 0:
+                        parts.append(" " * gap)
+            reconstructed_text = "".join(parts)
+
+            return Entity(
+                text=reconstructed_text,
+                entity_type=entity_type,
+                start_idx=start_offset,
+                end_idx=end_offset,
+            )
+
+        while i < n:
+            token = tokens[i]
+
+            # 1. Numeric Date check (e.g. 15-08-1947 or 15/08/47)
+            if (
+                i + 4 < n
+                and tokens[i].text.isdigit()
+                and tokens[i + 1].text in ("-", "/", ".")
+                and tokens[i + 2].text.isdigit()
+                and tokens[i + 3].text in ("-", "/", ".")
+                and tokens[i + 4].text.isdigit()
+            ):
+                joined_date = "".join([t.text for t in tokens[i : i + 5]])
+                if self._numeric_date_pattern.match(joined_date):
+                    entities.append(make_entity(i, i + 5, "date"))
+                    i += 5
+                    continue
+
+            # 2. Standalone Weekday
+            if token.text in self._weekdays:
+                entities.append(make_entity(i, i + 1, "date"))
+                i += 1
+                continue
+
+            # 3. Numeric Time check (e.g. 10:30 or 10:30 pm)
+            if (
+                i + 2 < n
+                and tokens[i].text.isdigit()
+                and tokens[i + 1].text == ":"
+                and tokens[i + 2].text.isdigit()
+            ):
+                joined_time = "".join([t.text for t in tokens[i : i + 3]])
+                if i + 3 < n and tokens[i + 3].text.lower() in ("am", "pm"):
+                    joined_time_am_pm = f"{joined_time} {tokens[i + 3].text.lower()}"
+                    if self._numeric_time_pattern.match(joined_time_am_pm):
+                        entities.append(make_entity(i, i + 4, "time"))
+                        i += 4
+                        continue
+                if self._numeric_time_pattern.match(joined_time):
+                    entities.append(make_entity(i, i + 3, "time"))
+                    i += 3
+                    continue
+
+            # Retrieve number sequences
+            num_len = self._get_num_sequence_len(tokens, i)
+
+            # Check for patterns starting with a number sequence
+            if num_len > 0:
+                next_idx = i + num_len
+
+                # A. Currency: [NumSequence] [Currency] [NumSequence] [Currency]
+                if (
+                    next_idx + 2 < n
+                    and tokens[next_idx].text.lower() in self._currency_words
+                    and self._is_num(tokens[next_idx + 1])
+                ):
+                    sub_num_len = self._get_num_sequence_len(tokens, next_idx + 1)
+                    curr2_idx = next_idx + 1 + sub_num_len
+                    if (
+                        curr2_idx < n
+                        and tokens[curr2_idx].text.lower() in self._currency_words
+                    ):
+                        entities.append(make_entity(i, curr2_idx + 1, "currency"))
+                        i = curr2_idx + 1
+                        continue
+
+                # B. Currency: [NumSequence] [Currency]
+                if (
+                    next_idx < n
+                    and tokens[next_idx].text.lower() in self._currency_words
+                ):
+                    entities.append(make_entity(i, next_idx + 1, "currency"))
+                    i = next_idx + 1
+                    continue
+
+                # C. Time: [NumSequence] बजकर [NumSequence] मिनट
+                if (
+                    next_idx + 2 < n
+                    and tokens[next_idx].text == "बजकर"
+                    and self._is_num(tokens[next_idx + 1])
+                ):
+                    sub_num_len = self._get_num_sequence_len(tokens, next_idx + 1)
+                    min_idx = next_idx + 1 + sub_num_len
+                    if min_idx < n and tokens[min_idx].text == "मिनट":
+                        entities.append(make_entity(i, min_idx + 1, "time"))
+                        i = min_idx + 1
+                        continue
+
+                # D. Time: [NumSequence] [TimeWord]
+                if next_idx < n and tokens[next_idx].text.lower() in self._time_words:
+                    entities.append(make_entity(i, next_idx + 1, "time"))
+                    i = next_idx + 1
+                    continue
+
+                # E. Date: [NumSequence] [Month] [NumSequence]
+                if (
+                    next_idx + 1 < n
+                    and tokens[next_idx].text in self._months
+                    and self._is_num(tokens[next_idx + 1])
+                ):
+                    sub_num_len = self._get_num_sequence_len(tokens, next_idx + 1)
+                    entities.append(make_entity(i, next_idx + 1 + sub_num_len, "date"))
+                    i = next_idx + 1 + sub_num_len
+                    continue
+
+                # F. Date: [NumSequence] [Month]
+                if next_idx < n and tokens[next_idx].text in self._months:
+                    entities.append(make_entity(i, next_idx + 1, "date"))
+                    i = next_idx + 1
+                    continue
+
+                # G. Decimal: [NumSequence] [DecimalIndicator] [NumSequence]
+                if (
+                    next_idx + 1 < n
+                    and tokens[next_idx].text.lower() in self._decimal_indicators
+                    and self._is_num(tokens[next_idx + 1])
+                ):
+                    sub_num_len = self._get_num_sequence_len(tokens, next_idx + 1)
+                    entities.append(
+                        make_entity(i, next_idx + 1 + sub_num_len, "decimal")
+                    )
+                    i = next_idx + 1 + sub_num_len
+                    continue
+
+                # H. Percentage: [NumSequence] [PercentageIndicator]
+                if (
+                    next_idx < n
+                    and tokens[next_idx].text.lower() in self._percentage_indicators
+                ):
+                    entities.append(make_entity(i, next_idx + 1, "percentage"))
+                    i = next_idx + 1
+                    continue
+
+                # I. Time: [NumSequence] [Period] (e.g. पाँच बीस पी एम, पाँच पी एम)
+                period_len = self._get_period_len(tokens, next_idx)
+                if period_len > 0:
+                    entities.append(make_entity(i, next_idx + period_len, "time"))
+                    i = next_idx + period_len
+                    continue
+
+            # Check for patterns starting with a modifier or keyword
+            if token.text in self._time_modifiers:
+                # Case A: [Modifier] [NumSequence] [TimeWord] (e.g., साढ़े पांच बजे)
+                if i + 1 < n and self._is_num(tokens[i + 1]):
+                    sub_num_len = self._get_num_sequence_len(tokens, i + 1)
+                    t_idx = i + 1 + sub_num_len
+                    if t_idx < n and tokens[t_idx].text.lower() in self._time_words:
+                        entities.append(make_entity(i, t_idx + 1, "time"))
+                        i = t_idx + 1
+                        continue
+
+                # Case B: [Modifier] [TimeWord] (e.g., डेढ़ बजे, ढाई बजे)
+                if i + 1 < n and tokens[i + 1].text.lower() in self._time_words:
+                    entities.append(make_entity(i, i + 2, "time"))
+                    i = i + 2
+                    continue
+
+                # Case C: [Modifier] [NumSequence] only (e.g., साढ़े पांच, सवा पांच, पौने दो)
+                if (
+                    token.text in {"साढ़े", "साढ़े", "सवा", "पौने"}
+                    and i + 1 < n
+                    and self._is_num(tokens[i + 1])
+                ):
+                    sub_num_len = self._get_num_sequence_len(tokens, i + 1)
+                    entities.append(make_entity(i, i + 1 + sub_num_len, "time"))
+                    i = i + 1 + sub_num_len
+                    continue
+
+            # H. Date: [Month] [NumSequence]
+            if token.text in self._months and i + 1 < n and self._is_num(tokens[i + 1]):
+                sub_num_len = self._get_num_sequence_len(tokens, i + 1)
+                entities.append(make_entity(i, i + 1 + sub_num_len, "date"))
+                i = i + 1 + sub_num_len
+                continue
+
+            # 4. Fallback: process plain number sequences
+            if num_len > 0:
+                span_tokens = tokens[i : i + num_len]
+                # Determine classification based on digit-like structure
+                digit_count = 0
+                all_single_digits = True
+
+                for t in span_tokens:
+                    if t.text.isdigit():
+                        digit_count += len(t.text)
+                        if len(t.text) > 1:
+                            all_single_digits = False
+                    elif t.text in self._digits:
+                        digit_count += 1
+                    else:
+                        digit_count += 1
+                        all_single_digits = False
+
+                # Check for phone context
+                has_phone_context = False
+                for check_idx in (i - 1, i - 2):
+                    if check_idx >= 0:
+                        context_word = tokens[check_idx].text.lower()
+                        if context_word in {
+                            "फोन",
+                            "मोबाइल",
+                            "नंबर",
+                            "नम्बर",
+                            "सम्पर्क",
+                            "संपर्क",
+                            "phone",
+                            "mobile",
+                            "number",
+                            "contact",
+                        }:
+                            has_phone_context = True
+                            break
+
+                # Check for OTP context
+                has_otp_context = False
+                for check_idx in (i - 1, i - 2):
+                    if check_idx >= 0:
+                        word = tokens[check_idx].text.lower()
+                        if word in {"otp", "ओटीपी", "pin", "पिन"}:
+                            has_otp_context = True
+                            break
+                if not has_otp_context and i - 2 >= 0:
+                    phrase = (
+                        f"{tokens[i - 2].text.lower()} {tokens[i - 1].text.lower()}"
+                    )
+                    if phrase in {
+                        "verification code",
+                        "security code",
+                        "वेरिफिकेशन कोड",
+                        "सिक्योरिटी कोड",
+                    }:
+                        has_otp_context = True
+
+                # Apply length classification heuristic
+                if digit_count == 10 or (has_phone_context and 8 <= digit_count <= 13):
+                    entities.append(make_entity(i, i + num_len, "phone"))
+                elif (4 <= digit_count <= 8 and all_single_digits) or (
+                    has_otp_context and 4 <= digit_count <= 8
+                ):
+                    entities.append(make_entity(i, i + num_len, "otp"))
+                else:
+                    entities.append(make_entity(i, i + num_len, "number"))
+
+                i += num_len
+                continue
+
+            # If no pattern matches, move to the next token
+            i += 1
+
+        return entities

indic_itn/loader.py

@@ -0,0 +1,21 @@
+"""Utility module to load packages resources."""
+
+import json
+from importlib.resources import files
+from typing import Any
+
+
+def load_hindi_resource(name: str) -> dict[str, Any]:
+    """Load a Devanagari Hindi JSON resource dictionary.
+
+    Args:
+        name: The filename of the resource without extension (e.g., "numbers",
+            "temporal", "ordinals").
+
+    Returns:
+        A dictionary containing the parsed resource content.
+    """
+    resource_path = files("indic_itn").joinpath("resources", "hi", f"{name}.json")
+    with resource_path.open(encoding="utf-8") as f:
+        data: dict[str, Any] = json.load(f)
+    return data