sinlib 0.1.13__tar.gz → 0.2.0__tar.gz

sinlib-0.2.0/.github/workflows/test.yml

@@ -0,0 +1,32 @@
+name: Run Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    name: Test on Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]" -r requirements.txt
+
+      - name: Run test suite with pytest
+        run: |
+          pytest --tb=short -v

{sinlib-0.1.13 → sinlib-0.2.0}/.gitignore

@@ -11,3 +11,9 @@ experiments/*
 *.ipynb
 .qodo
 examples/examples.ipynb
+
+# Astro / Starlight
+website/dist/
+website/node_modules/
+website/.astro/
+

sinlib-0.2.0/.readthedocs.yaml

@@ -0,0 +1,11 @@
+version: 2
+
+build:
+  os: ubuntu-24.04
+  tools:
+    nodejs: "20"
+  commands:
+    - cd website && npm ci
+    - cd website && npm run build
+    - mkdir -p $READTHEDOCS_OUTPUT/html
+    - cp -r website/dist/* $READTHEDOCS_OUTPUT/html/

{sinlib-0.1.13 → sinlib-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sinlib
-Version: 0.1.13
+Version: 0.2.0
 Summary: Sinhala NLP Toolkit
 Project-URL: Code, https://github.com/Ransaka/sinlib
 Project-URL: Docs, https://sinlib.readthedocs.io

{sinlib-0.1.13 → sinlib-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sinlib"
-version = "0.1.13"
+version = "0.2.0"
 description = "Sinhala NLP Toolkit"
 authors = [
     { name = "Ransaka", email = "ransaka.ravihara@gmail.com" }

{sinlib-0.1.13 → sinlib-0.2.0}/src/sinlib/__init__.py

@@ -7,16 +7,16 @@ transliteration of Sinhala text, along with preprocessing utilities.
 Available Classes
 -----------------
 Tokenizer
-    Character-level tokenizer for Sinhala text.  Mirrors the HuggingFace
+    Character-level tokenizer for Sinhala text. Mirrors the HuggingFace
     ``PreTrainedTokenizer`` interface — use ``Tokenizer.from_pretrained()``
     to load the default pretrained vocabulary.
 
 BatchEncoding
-    Dict-like container returned by the tokenizer.  Supports attribute access
+    Dict-like container returned by the tokenizer. Supports attribute access
     (``enc.input_ids``, ``enc.attention_mask``) and dict-style access.
 
 TypoDetector
-    N-gram–based spell checker for Sinhala.  Use
+    N-gram–based spell checker for Sinhala. Use
     ``TypoDetector.from_pretrained()`` or instantiate directly.
 
 preprocessing
@@ -25,30 +25,29 @@ preprocessing
 
 Romanizer
     Converts Sinhala text to Roman (Latin) script.
-    *Currently disabled from the default namespace due to an ongoing fix.*
 
 Transliterator
     ML-based Sinhala → Roman transliteration using a pre-trained BiLSTM.
-    *Currently disabled from the default namespace due to an ongoing fix.*
+    *Currently disabled from default __all__ pending model vocabulary realignment.*
 """
 
 from typing import List
 
 from sinlib.encoding import BatchEncoding
 from sinlib.tokenizer import Tokenizer
+from sinlib.subword import SubwordTokenizer
 from sinlib.spellcheck import TypoDetector
-from sinlib.utils import preprocessing
-
-# These are still importable directly but excluded from __all__ while
-# their respective bugs are being tracked down.
-from sinlib.romanize import Romanizer          # noqa: F401
+from sinlib.romanize import Romanizer
 from sinlib.transliterate import Transliterator  # noqa: F401
+from sinlib.utils import preprocessing
 
 __all__: List[str] = [
     "BatchEncoding",
     "Tokenizer",
+    "SubwordTokenizer",
     "TypoDetector",
+    "Romanizer",
     "preprocessing",
 ]
 
-__version__ = "0.1.12"
+__version__ = "0.1.13"

{sinlib-0.1.13 → sinlib-0.2.0}/src/sinlib/encoding.py

@@ -24,6 +24,9 @@ class BatchEncoding:
     data : dict
         A dictionary whose values are lists of integers.  Expected keys are
         ``"input_ids"`` and optionally ``"attention_mask"``.
+    tensor_type : str, optional
+        If set, converts lists to tensors. Supported values: ``"pt"`` (PyTorch),
+        ``"tf"`` (TensorFlow), ``"np"`` (NumPy).
 
     Examples
     --------
@@ -36,35 +39,26 @@ class BatchEncoding:
     True
     """
 
-    def __init__(self, data: Dict[str, Any]) -> None:
+    def __init__(self, data: Dict[str, Any], tensor_type: Optional[str] = None) -> None:
         self._data: Dict[str, Any] = data
+        if tensor_type is not None:
+            self.convert_to_tensors(tensor_type)
 
     # ------------------------------------------------------------------
     # Attribute access
     # ------------------------------------------------------------------
 
     @property
-    def input_ids(self) -> List[int]:
+    def input_ids(self) -> Any:
         """
-        List of token IDs.
-
-        Returns
-        -------
-        list of int
-            The encoded token IDs for the input text.
+        Token IDs (list or tensor depending on return_tensors).
         """
         return self._data["input_ids"]
 
     @property
-    def attention_mask(self) -> Optional[List[int]]:
+    def attention_mask(self) -> Any:
         """
         Attention mask (1 for real tokens, 0 for padding).
-
-        Returns
-        -------
-        list of int or None
-            ``1`` for each real token position, ``0`` for padding.
-            ``None`` when the tokenizer was called without padding.
         """
         return self._data.get("attention_mask")
 
@@ -103,6 +97,51 @@ class BatchEncoding:
     # Conversion
     # ------------------------------------------------------------------
 
+    def convert_to_tensors(self, tensor_type: str) -> None:
+        """
+        Convert the internal input lists to tensors.
+        """
+        if tensor_type == "pt":
+            import torch
+            for key, val in self._data.items():
+                if val is not None:
+                    try:
+                        self._data[key] = torch.tensor(val)
+                    except ValueError as e:
+                        raise ValueError(
+                            f"Failed to convert {key} to PyTorch tensor. "
+                            f"Ensure sequences are rectangular (uniform length) or padding is enabled. "
+                            f"Error: {e}"
+                        ) from e
+        elif tensor_type == "tf":
+            import tensorflow as tf
+            for key, val in self._data.items():
+                if val is not None:
+                    try:
+                        self._data[key] = tf.convert_to_tensor(val)
+                    except (ValueError, TypeError) as e:
+                        raise ValueError(
+                            f"Failed to convert {key} to TensorFlow tensor. "
+                            f"Ensure sequences are rectangular (uniform length) or padding is enabled. "
+                            f"Error: {e}"
+                        ) from e
+        elif tensor_type == "np":
+            import numpy as np
+            for key, val in self._data.items():
+                if val is not None:
+                    try:
+                        self._data[key] = np.array(val)
+                    except ValueError as e:
+                        raise ValueError(
+                            f"Failed to convert {key} to NumPy array. "
+                            f"Ensure sequences are rectangular (uniform length) or padding is enabled. "
+                            f"Error: {e}"
+                        ) from e
+        else:
+            raise ValueError(
+                f"Unsupported tensor_type '{tensor_type}'. Use 'pt', 'tf', or 'np'."
+            )
+
     def to_dict(self) -> Dict[str, Any]:
         """
         Convert to a plain Python dictionary.

{sinlib-0.1.13 → sinlib-0.2.0}/src/sinlib/romanize.py

@@ -11,7 +11,7 @@ from numpy.typing import NDArray
 
 from .tokenizer import Tokenizer
 from .utils.chars import ALL_SINHALA_CHARACTERS, NUMBERS_AND_PUNCTUATION
-from .utils.preprocessing import load_char_mapper, remove_non_printable
+from .utils.preprocessing import load_char_mapper, remove_non_printable, process_text
 
 
 class Romanizer:
@@ -39,8 +39,7 @@ class Romanizer:
             tokenizer_path: Path to tokenizer vocabulary file
         """
         self.char_mapper = load_char_mapper()
-        self.tokenizer = Tokenizer(max_length=None)
-        self.tokenizer.load_from_pretrained(file_path=None, load_default_tokenizer=True)
+        self.tokenizer = Tokenizer.from_pretrained("Ransaka/sinlib")
 
     def __call__(self, text: Union[str, List[str]]) -> Union[str, List[str]]:
         """
@@ -67,32 +66,21 @@ class Romanizer:
             Romanized version of the input text
         """
         text = remove_non_printable(text)
-        chars: NDArray = np.array(list(text))
-        
-        # Create mask for Sinhala characters and allowed punctuation
-        sinhala_mask = [
-            char in ALL_SINHALA_CHARACTERS + list(NUMBERS_AND_PUNCTUATION) + [" "]
-            for char in chars
-        ]
-        
-        # Extract Sinhala text
-        sinhala_text = "".join(chars[sinhala_mask]).strip()
-        
-        # Tokenize and decode
-        encodings = self.tokenizer(sinhala_text, truncate_and_pad=False)
-        decoded_chars = [
-            self.tokenizer.token_id_to_token_map[c] for c in encodings
-        ]
-        
-        # Convert to Roman characters
-        romanized_chars = [
-            self.char_mapper.get(ch, ch if ch in NUMBERS_AND_PUNCTUATION.union({" "}) else '')
-            for ch in decoded_chars
-        ]
-        romanized_sinhala = "".join(romanized_chars)
-        
-        # Create word mapping and apply to full text
-        word_mapping = dict(zip(sinhala_text.split(), romanized_sinhala.split()))
-        romanized_words = [word_mapping.get(word, word) for word in text.split()]
-        
-        return " ".join(romanized_words)
+        if not text:
+            return ""
+
+        # Fallback mappings for diacritics/modifiers
+        char_map = dict(self.char_mapper)
+        char_map.setdefault('ං', 'n')
+        char_map.setdefault('ඃ', 'h')
+
+        tokens = process_text(text)
+        res = []
+        for tok in tokens:
+            clean = tok.strip()
+            roman = char_map.get(clean, clean)
+            if tok.endswith(' ') and not roman.endswith(' '):
+                roman += ' '
+            res.append(roman)
+
+        return "".join(res)