sinlib 0.3.0__tar.gz → 0.3.2__tar.gz

{sinlib-0.3.0 → sinlib-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: sinlib
-Version: 0.3.0
+Version: 0.3.2
 Summary: Sinhala NLP Toolkit
 Project-URL: Code, https://github.com/Ransaka/sinlib
 Project-URL: Docs, https://sinlib.readthedocs.io
@@ -147,9 +147,8 @@ Notes:
 
 - **Default behavior is unchanged** — without `neural_backend` the detector is
   purely statistical and requires no torch.
-- Neural corrections are gated: clean in-dictionary sentences never hit the
-  model, and a candidate is accepted only if it scores no worse than the input
-  (hallucination guard).
+- A neural candidate is accepted only if it scores no worse than the input
+  (hallucination guard), so already-clean text is never degraded.
 - If the checkpoint cannot be downloaded, the detector degrades gracefully to
   statistical-only correction with a warning.
 

{sinlib-0.3.0 → sinlib-0.3.2}/README.md

@@ -100,9 +100,8 @@ Notes:
 
 - **Default behavior is unchanged** — without `neural_backend` the detector is
   purely statistical and requires no torch.
-- Neural corrections are gated: clean in-dictionary sentences never hit the
-  model, and a candidate is accepted only if it scores no worse than the input
-  (hallucination guard).
+- A neural candidate is accepted only if it scores no worse than the input
+  (hallucination guard), so already-clean text is never degraded.
 - If the checkpoint cannot be downloaded, the detector degrades gracefully to
   statistical-only correction with a warning.
 

{sinlib-0.3.0 → sinlib-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sinlib"
-version = "0.3.0"
+version = "0.3.2"
 description = "Sinhala NLP Toolkit"
 authors = [
     { name = "Ransaka", email = "ransaka.ravihara@gmail.com" }

{sinlib-0.3.0 → sinlib-0.3.2}/src/sinlib/__init__.py

@@ -62,4 +62,4 @@ __all__: List[str] = [
     "setup_matplotlib",
 ]
 
-__version__ = "0.3.0"
+__version__ = "0.3.2"

{sinlib-0.3.0 → sinlib-0.3.2}/src/sinlib/charbert/encoder.py

@@ -120,6 +120,25 @@ class SinhalaCharBERTEncoder(nn.Module):
         all_char_hidden_states: List[torch.Tensor] = []
         all_attentions: List[torch.Tensor] = []
 
+        # Normalize the attention mask to the 4-D extended additive format
+        # (batch, 1, 1, seq_len) expected by transformers' attention
+        # implementations. Newer versions of transformers (>= 4.56) index the
+        # mask as attention_mask[:, :, :, :key_len] and crash with
+        # "too many indices for tensor of dimension 2" if a 2-D padding mask
+        # is passed straight through.
+        if attention_mask is not None and attention_mask.dim() != 4:
+            if attention_mask.dim() == 2:
+                mask = attention_mask[:, None, None, :]
+            elif attention_mask.dim() == 3:
+                mask = attention_mask[:, None, :, :]
+            else:
+                raise ValueError(
+                    f"Unsupported attention_mask dims: {attention_mask.dim()}"
+                )
+            # Convert 1/0 padding mask to an additive mask (0 keep, -inf drop)
+            dtype = token_hidden_states.dtype
+            attention_mask = (1.0 - mask.to(dtype)) * torch.finfo(dtype).min
+
         curr_token = token_hidden_states
         curr_char = char_hidden_states
 

{sinlib-0.3.0 → sinlib-0.3.2}/src/sinlib/charbert/seq2seq_decoder.py

@@ -78,7 +78,11 @@ class SinhalaCharBERTDecoderLayer(nn.Module):
             key=hidden_states,
             value=hidden_states,
             attn_mask=causal_mask,
-            need_weights=False,
+            # need_weights=True forces the math code path of
+            # nn.MultiheadAttention; the fused fast path has shown
+            # device-dependent failures with 2-D float causal masks
+            # on some CUDA builds.
+            need_weights=True,
         )
         hidden_states = self.self_attn_layer_norm(residual + self.dropout(attn_out))
 

{sinlib-0.3.0 → sinlib-0.3.2}/src/sinlib/spellcheck.py

@@ -4,6 +4,7 @@ import math
 import warnings
 import json
 import re
+import traceback
 from pathlib import Path
 from difflib import get_close_matches, SequenceMatcher
 from typing import Dict, List, Optional, Set, Tuple
@@ -663,22 +664,19 @@ class TypoDetector:
 
     def _try_neural_sentence(self, text: str) -> Optional[str]:
         """
-        Runs the CharBERT seq2seq neural pass over a sentence when structural
-        noise or unfixable suspicious words are detected. Returns the neural
-        correction if it passes the acceptance guard, otherwise None.
+        Runs the CharBERT seq2seq neural pass over a sentence and returns the
+        neural correction if it passes the acceptance guard, otherwise None.
+
+        The model runs on every non-empty sentence: real-word typos
+        (e.g. sibilant confusion) and word split/fusion artifacts consist of
+        perfectly valid dictionary words and cannot be detected by any cheap
+        statistical heuristic. The acceptance guard — the neural output must
+        score no worse than the input — is what protects already-clean text
+        from spurious "corrections".
         """
         if not text or not text.strip() or not self.has_charbert:
             return None
 
-        # Trigger gate: structural noise or any out-of-dictionary word
-        words = [self._extract_punctuation(t)[1] for t in text.split()]
-        trigger = CharBERTBackend.has_structural_noise(text) or any(
-            w and (w not in self._dictionary and normalize_sinhala(w) not in self._dictionary)
-            for w in words
-        )
-        if not trigger:
-            return None
-
         if text in self._neural_sentence_cache:
             return self._neural_sentence_cache[text]
 
@@ -687,8 +685,13 @@ class TypoDetector:
             candidate = self._charbert.correct_sentence(
                 text, num_beams=self._backend_num_beams
             ).strip()
-        except Exception as exc:
-            warnings.warn(f"CharBERT neural correction failed: {exc}", RuntimeWarning)
+        except Exception:
+            # Include the traceback: identical warnings are deduplicated by the
+            # warnings module, so a short message would hide repeated failures.
+            warnings.warn(
+                "CharBERT neural correction failed:\n" + traceback.format_exc(),
+                RuntimeWarning,
+            )
 
         result: Optional[str] = None
         if candidate and candidate != text:
@@ -934,8 +937,19 @@ class TypoDetector:
         using Phonological Trie search, BiGRU Seq2Seq, and Stupid Backoff LM.
         """
         self._ensure_loaded()
+
+        # Neural-first pass (seq2seq / hybrid modes): when structural noise or
+        # out-of-dictionary words are present, run the CharBERT model on the
+        # RAW input BEFORE the statistical word loop, so that word-level
+        # replacements cannot corrupt the sentence the model conditions on.
+        neural_done: Optional[str] = None
+        if self.neural_backend in ("seq2seq", "hybrid"):
+            neural_done = self._try_neural_sentence(text)
+
+        work_text = neural_done if neural_done is not None else text
+
         corrected: List[str] = []
-        raw_tokens = text.split() if isinstance(text, str) else [str(text)]
+        raw_tokens = work_text.split() if isinstance(work_text, str) else [str(work_text)]
         parsed_tokens = [self._extract_punctuation(t) for t in raw_tokens]
         words = [p[1] for p in parsed_tokens]
         n_words = len(words)
@@ -979,8 +993,9 @@ class TypoDetector:
 
         result_text = " ".join(corrected)
 
-        # Open-vocabulary neural sentence pass (seq2seq / hybrid modes)
-        if self.neural_backend in ("seq2seq", "hybrid"):
+        # Fallback neural pass (seq2seq / hybrid modes): only when the
+        # neural-first pass did not already handle this sentence.
+        if self.neural_backend in ("seq2seq", "hybrid") and neural_done is None:
             neural_sentence = self._try_neural_sentence(result_text)
             if neural_sentence is not None:
                 result_text = neural_sentence

{sinlib-0.3.0 → sinlib-0.3.2}/tests/test_charbert_backend.py

@@ -142,11 +142,12 @@ def test_neural_pass_applies_correction(neural_detector):
     assert result == "මම ගෙදර යන්න ඕනේ"
 
 
-def test_neural_pass_skipped_on_clean_in_dictionary_sentence(neural_detector):
-    # All words in dictionary and no structural noise -> no neural call
+def test_clean_in_dictionary_sentence_unchanged(neural_detector):
+    # Clean sentence: the model is consulted, but its (unchanged) output is
+    # rejected by the acceptance guard, so the text is returned as-is.
     result = neural_detector("ගෙදර පොත")
     backend = FakeCharBERTBackend.instances[0]
-    assert len(backend.sentence_calls) == 0
+    assert len(backend.sentence_calls) == 1
     assert result == "ගෙදර පොත"
 
 

{sinlib-0.3.0 → sinlib-0.3.2}/website/src/content/docs/api/spellcheck.mdx

@@ -47,6 +47,31 @@ Main spell checking and correction class. Detects typos using Akshara Trigram sc
       <td><span class="param-type">bool</span><span class="param-badge badge-optional">Optional</span></td>
       <td>If True, defers downloading/loading weights until first call.</td>
     </tr>
+    <tr>
+      <td><span class="param-name">neural_backend</span></td>
+      <td><span class="param-type">str | None</span><span class="param-badge badge-optional">Optional</span></td>
+      <td>Optional CharBERT neural correction mode: <code>None</code>, <code>&quot;denoise&quot;</code>, <code>&quot;seq2seq&quot;</code>, or <code>&quot;hybrid&quot;</code>. Default <code>None</code> (statistical only).</td>
+    </tr>
+    <tr>
+      <td><span class="param-name">backend_model</span></td>
+      <td><span class="param-type">str</span><span class="param-badge badge-optional">Optional</span></td>
+      <td>HF Hub repo id or local checkpoint directory for the CharBERT model. Default <code>&quot;Ransaka/sinhala-charbert-seq2seq&quot;</code>.</td>
+    </tr>
+    <tr>
+      <td><span class="param-name">backend_device</span></td>
+      <td><span class="param-type">str | None</span><span class="param-badge badge-optional">Optional</span></td>
+      <td>Torch device (cuda &gt; mps &gt; cpu by default).</td>
+    </tr>
+    <tr>
+      <td><span class="param-name">backend_revision</span></td>
+      <td><span class="param-type">str | None</span><span class="param-badge badge-optional">Optional</span></td>
+      <td>Pin a specific HF Hub revision of the CharBERT checkpoint.</td>
+    </tr>
+    <tr>
+      <td><span class="param-name">backend_num_beams</span></td>
+      <td><span class="param-type">int</span><span class="param-badge badge-optional">Optional</span></td>
+      <td>Beam width for seq2seq generation. Default <code>4</code>.</td>
+    </tr>
   </tbody>
 </table>
 
@@ -187,5 +212,26 @@ prob = detector.word_ngram_probability("සිංහල")
 # ~3.2e-05  (higher is more common)
 ```
 
+#### Neural Correction (CharBERT, optional)
+```python
+# pip install "sinlib[charbert]"
+detector = TypoDetector(neural_backend="hybrid")
+
+detector("මම ගෙදර යන්ඩ ඕනේ")
+# 'මම ගෙදර යන්න ඕනේ'  (dialectal morphology)
+
+detector("මම gedara යන්න ඕනේ")
+# 'මම ගෙදර යන්න ඕනේ'  (Singlish code-mixing)
+
+detector("ක්රීඩාව")
+# 'ක්‍රීඩාව'  (ZWJ-damaged ligature)
+
+# Point at a local checkpoint instead of the Hub
+detector = TypoDetector(
+    neural_backend="seq2seq",
+    backend_model="/path/to/checkpoint",
+)
+```
+
 </div>
 </div>

{sinlib-0.3.0 → sinlib-0.3.2}/website/src/content/docs/guides/spellcheck.mdx

@@ -101,9 +101,62 @@ words = detector.get_dictionary()   # set[str]
 probs = detector.get_ngram_probs()  # dict
 ```
 
+## Neural typo correction (optional, CharBERT)
+
+Version 0.3.0 adds an optional neural backend backed by **Sinhala-CharBERT** —
+a dual-channel (subword + phonological akshara) seq2seq model. It handles noise
+classes the statistical pipeline cannot fix: Singlish transliteration,
+dialectal morphology (`යන්ඩ` → `යන්න`), ZWJ-damaged ligatures
+(`ක්රීඩාව` → `ක්‍රීඩාව`), split/fused words, and Unicode decomposition errors.
+
+Install the optional dependency and pick a backend mode:
+
+```python
+from sinlib import TypoDetector
+
+# "denoise"  - bounded word-level neural fix when dictionary suggestions fail
+# "seq2seq"  - open-vocabulary sentence-level fix when structural noise is detected
+# "hybrid"   - both, in cascade (recommended)
+detector = TypoDetector(neural_backend="hybrid")
+
+detector("මම ගෙදර යන්ඩ ඕනේ")
+# 'මම ගෙදර යන්න ඕනේ'
+
+detector("මම gedara යන්න ඕනේ")
+# 'මම ගෙදර යන්න ඕනේ'
+
+detector("ක්රීඩාව")
+# 'ක්‍රීඩාව'
+```
+
+### Constructor keywords
+
+| Keyword | Default | Description |
+|---|---|---|
+| `neural_backend` | `None` | `None`, `"denoise"`, `"seq2seq"`, or `"hybrid"` |
+| `backend_model` | `Ransaka/sinhala-charbert-seq2seq` | HF Hub repo id or a local checkpoint directory containing `pytorch_model.bin` + `char_vocab.json` |
+| `backend_device` | auto (cuda > mps > cpu) | Torch device for the model |
+| `backend_revision` | `None` | Pin a specific HF Hub revision |
+| `backend_num_beams` | `4` | Beam width for seq2seq generation |
+
+### How the gates work
+
+- Without `neural_backend`, behaviour is unchanged and purely statistical — no
+  torch required.
+- In `"seq2seq"` / `"hybrid"` modes, the model runs on every sentence: real-word
+  typos (e.g. sibilant confusion) and word split/fusion artifacts consist of
+  valid dictionary words, so no cheap heuristic can pre-detect them. On GPU the
+  beam search pass takes well under a second per sentence.
+- A neural candidate is accepted only if it is no noisier than the input
+  (hallucination guard), so clean text is never "corrected" into garbage.
+- If the checkpoint cannot be downloaded, the detector degrades gracefully to
+  statistical-only correction with a warning.
+
 ## Limitations
 
-- Corrections are made word-by-word; the model has no sentence context.
+- Statistical correction is word-by-word; the model has no sentence context
+  (the optional neural backend does operate on full sentences).
 - Only the top suggestion replaces a typo — alternatives available via `suggest_correction()`.
 - The dictionary covers general Sinhala vocabulary; domain-specific or neologistic words may be flagged.
-- Romanised Sinhala (Singlish) is not supported.
+- Romanised Sinhala (Singlish) and structural noise are only handled by the
+  optional CharBERT backend described above.