sinlib 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sinlib-0.2.1 → sinlib-0.2.2}/PKG-INFO +2 -1
- {sinlib-0.2.1 → sinlib-0.2.2}/pyproject.toml +3 -2
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/__init__.py +9 -5
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/__init__.py +2 -0
- sinlib-0.2.2/src/sinlib/utils/visualize.py +32 -0
- sinlib-0.2.2/tests/test_visualize.py +15 -0
- sinlib-0.2.2/website/src/content/docs/api/visualization.mdx +49 -0
- sinlib-0.2.2/website/src/content/docs/examples/visualization.md +33 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/.github/workflows/python-publish.yml +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/.github/workflows/test.yml +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/.gitignore +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/.readthedocs.yaml +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/LICENSE +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/README.md +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/requirements-dev.txt +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/requirements.txt +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/encoding.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/romanize.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/spellcheck.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/subword.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/tokenizer.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/transliterate.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/chars.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/dataset_utils.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/model_utils.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/models/__init__.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/models/akshara_ngram.json +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/models/akshara_vocab.json +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/models/bigru_detector.pt +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/models/transliterator-checkpoint.pth +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/models/transliterator_model.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/src/sinlib/utils/preprocessing.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/tests/test_combined_detector.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/tests/test_phase3_enhancements.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/tests/test_romanizer.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/tests/test_spellcheck.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/tests/test_tokenizer.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/tests/test_tokenizer_batch.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/tests/test_tokenizer_bos.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/tests/test_tokenizer_special_tokens.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/tests/test_transliterator.py +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/.gitignore +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/.vscode/extensions.json +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/.vscode/launch.json +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/README.md +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/astro.config.mjs +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/package-lock.json +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/package.json +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/public/favicon.svg +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/assets/houston.webp +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/assets/logo-dark.png +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/assets/logo-light.png +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/api/encoding.mdx +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/api/preprocessing.mdx +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/api/romanizer.mdx +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/api/spellcheck.mdx +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/api/tokenizer.mdx +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/examples/tokenization.md +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/examples/typo-correction.md +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/guides/spellcheck.mdx +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/guides/tokenization.mdx +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content/docs/index.mdx +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/content.config.ts +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/src/styles/custom.css +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/website/tsconfig.json +0 -0
- {sinlib-0.2.1 → sinlib-0.2.2}/welcome.png +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sinlib
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Sinhala NLP Toolkit
|
|
5
5
|
Project-URL: Code, https://github.com/Ransaka/sinlib
|
|
6
6
|
Project-URL: Docs, https://sinlib.readthedocs.io
|
|
@@ -35,6 +35,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
35
35
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
36
36
|
Requires-Python: >=3.9.7
|
|
37
37
|
Requires-Dist: huggingface-hub
|
|
38
|
+
Requires-Dist: matplotlib>=3.9.4
|
|
38
39
|
Requires-Dist: numpy>=1.24.0
|
|
39
40
|
Requires-Dist: tqdm>=4.64.1
|
|
40
41
|
Requires-Dist: transformers>=4.31.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sinlib"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.2"
|
|
4
4
|
description = "Sinhala NLP Toolkit"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Ransaka", email = "ransaka.ravihara@gmail.com" }
|
|
@@ -9,7 +9,8 @@ dependencies = [
|
|
|
9
9
|
"numpy >= 1.24.0",
|
|
10
10
|
"transformers >= 4.31.0",
|
|
11
11
|
"tqdm >= 4.64.1",
|
|
12
|
-
"huggingface_hub"
|
|
12
|
+
"huggingface_hub",
|
|
13
|
+
"matplotlib>=3.9.4",
|
|
13
14
|
]
|
|
14
15
|
license = { file = "LICENSE" }
|
|
15
16
|
readme = "README.md"
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
Sinlib: A comprehensive library for Sinhala text processing.
|
|
3
3
|
|
|
4
4
|
Provides tools for tokenization, spell-checking, romanization, and
|
|
5
|
-
transliteration of Sinhala text, along with preprocessing utilities.
|
|
5
|
+
transliteration of Sinhala text, along with preprocessing and visualization utilities.
|
|
6
6
|
|
|
7
|
-
Available Classes
|
|
8
|
-
|
|
7
|
+
Available Classes & Functions
|
|
8
|
+
-----------------------------
|
|
9
9
|
Tokenizer
|
|
10
10
|
Character-level tokenizer for Sinhala text. Mirrors the HuggingFace
|
|
11
11
|
``PreTrainedTokenizer`` interface — use ``Tokenizer.from_pretrained()``
|
|
@@ -23,6 +23,9 @@ preprocessing
|
|
|
23
23
|
Utility module exposing ``remove_english_characters``,
|
|
24
24
|
``remove_non_printable``, and ``get_sinhala_character_ratio``.
|
|
25
25
|
|
|
26
|
+
setup_matplotlib
|
|
27
|
+
Configures and returns a FontProperties object for Matplotlib rendering.
|
|
28
|
+
|
|
26
29
|
Romanizer
|
|
27
30
|
Converts Sinhala text to Roman (Latin) script.
|
|
28
31
|
|
|
@@ -39,7 +42,7 @@ from sinlib.subword import SubwordTokenizer
|
|
|
39
42
|
from sinlib.spellcheck import TypoDetector
|
|
40
43
|
from sinlib.romanize import Romanizer
|
|
41
44
|
from sinlib.transliterate import Transliterator # noqa: F401
|
|
42
|
-
from sinlib.utils import preprocessing
|
|
45
|
+
from sinlib.utils import preprocessing, setup_matplotlib
|
|
43
46
|
|
|
44
47
|
__all__: List[str] = [
|
|
45
48
|
"BatchEncoding",
|
|
@@ -48,6 +51,7 @@ __all__: List[str] = [
|
|
|
48
51
|
"TypoDetector",
|
|
49
52
|
"Romanizer",
|
|
50
53
|
"preprocessing",
|
|
54
|
+
"setup_matplotlib",
|
|
51
55
|
]
|
|
52
56
|
|
|
53
|
-
__version__ = "0.2.
|
|
57
|
+
__version__ = "0.2.2"
|
|
@@ -4,10 +4,12 @@ from .preprocessing import (
|
|
|
4
4
|
get_sinhala_character_ratio,
|
|
5
5
|
normalize_sinhala,
|
|
6
6
|
)
|
|
7
|
+
from .visualize import setup_matplotlib
|
|
7
8
|
|
|
8
9
|
__all__ = [
|
|
9
10
|
"remove_english_characters",
|
|
10
11
|
"remove_non_printable",
|
|
11
12
|
"get_sinhala_character_ratio",
|
|
12
13
|
"normalize_sinhala",
|
|
14
|
+
"setup_matplotlib",
|
|
13
15
|
]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import matplotlib.font_manager as fm
|
|
2
|
+
from huggingface_hub import hf_hub_download
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def setup_matplotlib() -> fm.FontProperties:
|
|
6
|
+
"""Download Noto Sans Sinhala font and return a Matplotlib FontProperties object.
|
|
7
|
+
|
|
8
|
+
This font supports both Latin (English) and Sinhala Unicode blocks natively,
|
|
9
|
+
allowing mixed-script strings to be rendered cleanly in a single text element.
|
|
10
|
+
|
|
11
|
+
Returns:
|
|
12
|
+
matplotlib.font_manager.FontProperties bound to the cached Noto Sans Sinhala font.
|
|
13
|
+
|
|
14
|
+
Example:
|
|
15
|
+
>>> import matplotlib.pyplot as plt
|
|
16
|
+
>>> from sinlib import setup_matplotlib
|
|
17
|
+
>>> fp = setup_matplotlib()
|
|
18
|
+
>>> plt.text(0.5, 0.5, "Target Token: 'සිංහල'", fontproperties=fp)
|
|
19
|
+
"""
|
|
20
|
+
try:
|
|
21
|
+
font_path = hf_hub_download(
|
|
22
|
+
repo_id="Ransaka/sinlib",
|
|
23
|
+
filename="fonts/NotoSansSinhala-Regular.ttf",
|
|
24
|
+
repo_type="model",
|
|
25
|
+
)
|
|
26
|
+
except Exception as e:
|
|
27
|
+
raise RuntimeError(
|
|
28
|
+
"Could not download Noto Sans Sinhala font from Hugging Face Hub. "
|
|
29
|
+
f"Error: {e}"
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
return fm.FontProperties(fname=font_path)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import matplotlib.font_manager as fm
|
|
3
|
+
from sinlib import setup_matplotlib
|
|
4
|
+
from sinlib.utils import setup_matplotlib as utils_setup_matplotlib
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_setup_matplotlib_import() -> None:
|
|
8
|
+
assert setup_matplotlib is utils_setup_matplotlib
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_setup_matplotlib_returns_font_properties() -> None:
|
|
12
|
+
fp = setup_matplotlib()
|
|
13
|
+
assert isinstance(fp, fm.FontProperties)
|
|
14
|
+
assert fp.get_name() == "Noto Sans Sinhala"
|
|
15
|
+
assert os.path.exists(fp.get_file())
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Visualization
|
|
3
|
+
description: API reference for sinlib's Matplotlib Sinhala text visualization utilities.
|
|
4
|
+
tableOfContents: false
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
Utility function for setting up zero-configuration Sinhala text rendering in Matplotlib.
|
|
8
|
+
|
|
9
|
+
<div class="api-two-column">
|
|
10
|
+
<div class="api-desc-col">
|
|
11
|
+
|
|
12
|
+
<span class="api-tag tag-method">Function</span>
|
|
13
|
+
|
|
14
|
+
## setup\_matplotlib
|
|
15
|
+
|
|
16
|
+
Downloads Noto Sans Sinhala from the Hugging Face Hub (if not already cached) and returns a Matplotlib `FontProperties` object configured for Sinhala + English rendering.
|
|
17
|
+
|
|
18
|
+
<div class="api-signature">
|
|
19
|
+
{`setup_matplotlib()`}
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
### Returns
|
|
23
|
+
- `matplotlib.font_manager.FontProperties`: Font object to pass to Matplotlib text rendering calls via `fontproperties=fp`.
|
|
24
|
+
|
|
25
|
+
</div>
|
|
26
|
+
|
|
27
|
+
<div class="api-code-col">
|
|
28
|
+
|
|
29
|
+
### Code Example
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
import matplotlib.pyplot as plt
|
|
33
|
+
from sinlib import setup_matplotlib
|
|
34
|
+
|
|
35
|
+
fp = setup_matplotlib()
|
|
36
|
+
|
|
37
|
+
fig, ax = plt.subplots()
|
|
38
|
+
ax.text(
|
|
39
|
+
0.5, 0.5,
|
|
40
|
+
"Target Token: 'සිංහල'",
|
|
41
|
+
fontsize=20,
|
|
42
|
+
ha="center",
|
|
43
|
+
fontproperties=fp
|
|
44
|
+
)
|
|
45
|
+
plt.savefig("output.png")
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
</div>
|
|
49
|
+
</div>
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Matplotlib Sinhala Text Visualization
|
|
2
|
+
|
|
3
|
+
`sinlib` provides a zero-configuration utility `setup_matplotlib()` to resolve and configure Sinhala text rendering in Matplotlib.
|
|
4
|
+
|
|
5
|
+
## Quickstart
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
import matplotlib.pyplot as plt
|
|
9
|
+
from sinlib import setup_matplotlib
|
|
10
|
+
|
|
11
|
+
# Fetch and prepare font properties (Noto Sans Sinhala)
|
|
12
|
+
fp = setup_matplotlib()
|
|
13
|
+
|
|
14
|
+
# Create a plot with mixed English and Sinhala text
|
|
15
|
+
fig, ax = plt.subplots(figsize=(8, 3))
|
|
16
|
+
ax.set_title("OCR Model Attention Visualizer", fontsize=14, fontproperties=fp)
|
|
17
|
+
ax.text(
|
|
18
|
+
0.5, 0.5,
|
|
19
|
+
"Predicted Text: 'ඇසළ පුර පසලොස්වක පොහෝ දිනය අදයි'",
|
|
20
|
+
fontsize=16,
|
|
21
|
+
ha="center",
|
|
22
|
+
va="center",
|
|
23
|
+
fontproperties=fp
|
|
24
|
+
)
|
|
25
|
+
ax.axis("off")
|
|
26
|
+
plt.savefig("attention_vis.png", dpi=200, bbox_inches="tight")
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Features
|
|
30
|
+
|
|
31
|
+
- **Mixed-Script Support:** Noto Sans Sinhala covers both Latin (ASCII/English) and Sinhala codepoints, so strings containing both languages render cleanly in a single `plt.text()` call.
|
|
32
|
+
- **Automatic Caching:** Downloads the font from Hugging Face Hub on first run and caches it locally via `huggingface_hub`.
|
|
33
|
+
- **No Global Mutation:** Returns a clean `FontProperties` instance instead of mutating Matplotlib's global `rcParams`.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|