nepali_unicoder 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nepali_unicoder-0.3.0/MANIFEST.in +1 -0
- {nepali_unicoder-0.2.0/src/nepali_unicoder.egg-info → nepali_unicoder-0.3.0}/PKG-INFO +9 -4
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/README.md +6 -1
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/pyproject.toml +3 -3
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/convert.py +4 -2
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/data/preeti_rules.json +5 -1
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/engine.py +59 -28
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/loader.py +12 -31
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/tokenizer.py +39 -39
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/trie.py +5 -7
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0/src/nepali_unicoder.egg-info}/PKG-INFO +9 -4
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder.egg-info/SOURCES.txt +0 -1
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/tests/test_components.py +27 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/tests/test_conversion.py +16 -1
- nepali_unicoder-0.2.0/MANIFEST.in +0 -2
- nepali_unicoder-0.2.0/src/nepali_unicoder/rules.py +0 -2
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/LICENSE +0 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/setup.cfg +0 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/__init__.py +0 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/__main__.py +0 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/data/roman_rules.json +0 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder/data/word_maps.json +0 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder.egg-info/dependency_links.txt +0 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder.egg-info/entry_points.txt +0 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder.egg-info/requires.txt +0 -0
- {nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
recursive-include src/nepali_unicoder/data *.json
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
|
-
Name:
|
|
3
|
-
Version: 0.
|
|
2
|
+
Name: nepali-unicoder
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A Python package for Roman to Nepali (Devanagari) transliteration
|
|
5
5
|
Author-email: realsanjeev <075bei033.sanjeev@pcampus.edu.np>
|
|
6
6
|
License: MIT
|
|
@@ -11,7 +11,7 @@ Keywords: nepali,unicoder,roman to devanagari,nepali transliteration,preeti to u
|
|
|
11
11
|
Classifier: Programming Language :: Python
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
13
|
Classifier: Operating System :: OS Independent
|
|
14
|
-
Requires-Python: >=3.
|
|
14
|
+
Requires-Python: >=3.8
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
16
|
License-File: LICENSE
|
|
17
17
|
Provides-Extra: dev
|
|
@@ -36,7 +36,7 @@ Read the **[Full Documentation](https://realsanjeev.github.io/nepali_unicoder/)*
|
|
|
36
36
|
- **Smart Vowel Handling**: Distinguishes between independent vowels (e.g., 'aa' -> 'आ') and vowel signs/matras (e.g., 'ka' -> 'क', 'kaa' -> 'का').
|
|
37
37
|
- **Contextual Rules**: Handles complex Devanagari rules like reph positioning, matra reordering, and special character combinations.
|
|
38
38
|
- **Mixed Content Support**: Allows keeping English words or specific text in Roman script using `{}` blocks.
|
|
39
|
-
- **Customizable**: Supports custom word-level overrides via `word_maps.json`.
|
|
39
|
+
- **Customizable**: Supports custom word-level overrides via `word_maps.json` or `Converter(word_map={...})`.
|
|
40
40
|
- **CLI Support**: Can be used directly from the command line.
|
|
41
41
|
|
|
42
42
|
## Installation
|
|
@@ -83,6 +83,11 @@ print(converter.convert(text))
|
|
|
83
83
|
mixed_text = "mero naam {Sanjeev} ho"
|
|
84
84
|
print(converter.convert(mixed_text))
|
|
85
85
|
# Output: मेरो नाम् Sanjeev हो
|
|
86
|
+
|
|
87
|
+
# Your own whole-word mappings (override the bundled ones)
|
|
88
|
+
converter = Converter(word_map={"sanjeev": "सञ्जीव"})
|
|
89
|
+
print(converter.convert("Sanjeev"))
|
|
90
|
+
# Output: सञ्जीव
|
|
86
91
|
```
|
|
87
92
|
|
|
88
93
|
### Preeti Mode
|
|
@@ -14,7 +14,7 @@ Read the **[Full Documentation](https://realsanjeev.github.io/nepali_unicoder/)*
|
|
|
14
14
|
- **Smart Vowel Handling**: Distinguishes between independent vowels (e.g., 'aa' -> 'आ') and vowel signs/matras (e.g., 'ka' -> 'क', 'kaa' -> 'का').
|
|
15
15
|
- **Contextual Rules**: Handles complex Devanagari rules like reph positioning, matra reordering, and special character combinations.
|
|
16
16
|
- **Mixed Content Support**: Allows keeping English words or specific text in Roman script using `{}` blocks.
|
|
17
|
-
- **Customizable**: Supports custom word-level overrides via `word_maps.json`.
|
|
17
|
+
- **Customizable**: Supports custom word-level overrides via `word_maps.json` or `Converter(word_map={...})`.
|
|
18
18
|
- **CLI Support**: Can be used directly from the command line.
|
|
19
19
|
|
|
20
20
|
## Installation
|
|
@@ -61,6 +61,11 @@ print(converter.convert(text))
|
|
|
61
61
|
mixed_text = "mero naam {Sanjeev} ho"
|
|
62
62
|
print(converter.convert(mixed_text))
|
|
63
63
|
# Output: मेरो नाम् Sanjeev हो
|
|
64
|
+
|
|
65
|
+
# Your own whole-word mappings (override the bundled ones)
|
|
66
|
+
converter = Converter(word_map={"sanjeev": "सञ्जीव"})
|
|
67
|
+
print(converter.convert("Sanjeev"))
|
|
68
|
+
# Output: सञ्जीव
|
|
64
69
|
```
|
|
65
70
|
|
|
66
71
|
### Preeti Mode
|
|
@@ -3,15 +3,15 @@ requires = ["setuptools>=61.0"]
|
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
|
-
name = "
|
|
7
|
-
version = "0.
|
|
6
|
+
name = "nepali-unicoder"
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "A Python package for Roman to Nepali (Devanagari) transliteration"
|
|
9
9
|
authors = [
|
|
10
10
|
{ name="realsanjeev", email="075bei033.sanjeev@pcampus.edu.np" },
|
|
11
11
|
]
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
license = { text = "MIT" }
|
|
14
|
-
requires-python = ">=3.
|
|
14
|
+
requires-python = ">=3.8"
|
|
15
15
|
dependencies = []
|
|
16
16
|
classifiers = [
|
|
17
17
|
"Programming Language :: Python",
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
from nepali_unicoder.engine import Engine
|
|
2
4
|
|
|
3
5
|
|
|
@@ -6,8 +8,8 @@ class Converter:
|
|
|
6
8
|
Wrapper around Engine for backward compatibility.
|
|
7
9
|
"""
|
|
8
10
|
|
|
9
|
-
def __init__(self, mode: str = "roman"):
|
|
10
|
-
self.engine = Engine(mode=mode)
|
|
11
|
+
def __init__(self, mode: str = "roman", word_map: dict[str, str] | None = None):
|
|
12
|
+
self.engine = Engine(mode=mode, word_map=word_map)
|
|
11
13
|
|
|
12
14
|
def convert(self, text: str) -> str:
|
|
13
15
|
return self.engine.transliterate(text)
|
|
@@ -1,46 +1,68 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import re
|
|
2
|
-
from
|
|
4
|
+
from functools import lru_cache
|
|
3
5
|
|
|
4
|
-
from nepali_unicoder.loader import
|
|
5
|
-
|
|
6
|
+
from nepali_unicoder.loader import (
|
|
7
|
+
PreetiLoader,
|
|
8
|
+
RuleLoader,
|
|
9
|
+
load_json_data,
|
|
10
|
+
with_capitalized,
|
|
11
|
+
)
|
|
12
|
+
from nepali_unicoder.tokenizer import Token, Tokenizer
|
|
6
13
|
from nepali_unicoder.trie import Trie
|
|
7
14
|
|
|
15
|
+
_RE_WORD = re.compile(r"[A-Za-z]+")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@lru_cache(maxsize=None)
|
|
19
|
+
def _load_defaults(mode: str):
|
|
20
|
+
"""Build a mode's (trie, post_rules, word_map) once per process.
|
|
21
|
+
Shared by every Engine, so treat them as read-only."""
|
|
22
|
+
if mode == "preeti":
|
|
23
|
+
loader = PreetiLoader()
|
|
24
|
+
post_rules = tuple(
|
|
25
|
+
(re.compile(pattern, re.MULTILINE), replacement)
|
|
26
|
+
for pattern, replacement in loader.get_post_rules()
|
|
27
|
+
)
|
|
28
|
+
return loader.load(), post_rules, {}
|
|
29
|
+
word_map = with_capitalized(load_json_data("word_maps.json"))
|
|
30
|
+
return RuleLoader().load(), (), word_map
|
|
31
|
+
|
|
8
32
|
|
|
9
33
|
class Engine:
|
|
10
|
-
VALID_MODES = {"roman", "preeti"}
|
|
34
|
+
VALID_MODES = frozenset({"roman", "preeti"})
|
|
11
35
|
|
|
12
36
|
def __init__(
|
|
13
37
|
self,
|
|
14
|
-
trie:
|
|
15
|
-
tokenizer:
|
|
38
|
+
trie: Trie | None = None,
|
|
39
|
+
tokenizer: Tokenizer | None = None,
|
|
16
40
|
mode: str = "roman",
|
|
41
|
+
word_map: dict[str, str] | None = None,
|
|
17
42
|
):
|
|
18
43
|
if mode not in self.VALID_MODES:
|
|
19
44
|
raise ValueError(
|
|
20
45
|
f"Invalid mode {mode!r}. Expected one of {sorted(self.VALID_MODES)}"
|
|
21
46
|
)
|
|
47
|
+
if word_map:
|
|
48
|
+
if mode != "roman":
|
|
49
|
+
raise ValueError("word_map is only supported in 'roman' mode")
|
|
50
|
+
bad = [key for key in word_map if not _RE_WORD.fullmatch(key)]
|
|
51
|
+
if bad:
|
|
52
|
+
raise ValueError(
|
|
53
|
+
f"word_map keys must be single words of ASCII letters: {bad!r}"
|
|
54
|
+
)
|
|
22
55
|
self.mode = mode
|
|
23
|
-
self.post_rules = []
|
|
24
56
|
|
|
25
57
|
if trie is None:
|
|
26
|
-
|
|
27
|
-
loader = PreetiLoader()
|
|
28
|
-
# Load post-processing rules for Preeti mode
|
|
29
|
-
raw_rules = loader.get_post_rules()
|
|
30
|
-
self.post_rules = [
|
|
31
|
-
(re.compile(pattern), replacement)
|
|
32
|
-
for pattern, replacement in raw_rules
|
|
33
|
-
]
|
|
34
|
-
else:
|
|
35
|
-
loader = RuleLoader()
|
|
36
|
-
self.trie = loader.load()
|
|
58
|
+
self.trie, post_rules, default_map = _load_defaults(mode)
|
|
37
59
|
else:
|
|
38
|
-
self.trie = trie
|
|
60
|
+
self.trie, post_rules, default_map = trie, (), {}
|
|
61
|
+
self.post_rules = post_rules
|
|
62
|
+
# User entries win over the bundled ones; copy so the cache stays untouched.
|
|
63
|
+
self.word_map = {**default_map, **with_capitalized(word_map or {})}
|
|
39
64
|
|
|
40
|
-
|
|
41
|
-
self.tokenizer = Tokenizer()
|
|
42
|
-
else:
|
|
43
|
-
self.tokenizer = tokenizer
|
|
65
|
+
self.tokenizer = tokenizer or Tokenizer()
|
|
44
66
|
|
|
45
67
|
def transliterate(self, text: str) -> str:
|
|
46
68
|
"""
|
|
@@ -49,14 +71,15 @@ class Engine:
|
|
|
49
71
|
if not text:
|
|
50
72
|
return ""
|
|
51
73
|
|
|
52
|
-
|
|
53
|
-
|
|
74
|
+
if self.mode == "preeti":
|
|
75
|
+
# Every Preeti key (digits, '.', braces) is a glyph: map the whole text.
|
|
76
|
+
tokens = [Token(value=text, type="ROMAN")]
|
|
77
|
+
else:
|
|
78
|
+
tokens = self.tokenizer.tokenize(text)
|
|
54
79
|
result = []
|
|
55
80
|
|
|
56
81
|
for token in tokens:
|
|
57
|
-
if token.type == "BLOCK":
|
|
58
|
-
result.append(token.value)
|
|
59
|
-
elif token.type == "LITERAL":
|
|
82
|
+
if token.type == "BLOCK" or token.type == "LITERAL":
|
|
60
83
|
result.append(token.value)
|
|
61
84
|
elif token.type == "NUMBER":
|
|
62
85
|
# Process Number chunk: transliterate digits, keep others (like .) as is
|
|
@@ -77,6 +100,14 @@ class Engine:
|
|
|
77
100
|
chunk_len = len(chunk)
|
|
78
101
|
|
|
79
102
|
while idx < chunk_len:
|
|
103
|
+
# Word maps apply to whole words only ('bus' but not 'bushan')
|
|
104
|
+
if self.word_map and (idx == 0 or not chunk[idx - 1].isalpha()):
|
|
105
|
+
word = _RE_WORD.match(chunk, idx)
|
|
106
|
+
if word and word.group() in self.word_map:
|
|
107
|
+
result.append(self.word_map[word.group()])
|
|
108
|
+
idx = word.end()
|
|
109
|
+
continue
|
|
110
|
+
|
|
80
111
|
# Try to find longest match starting at idx
|
|
81
112
|
# Pass chunk and idx directly to avoid slicing
|
|
82
113
|
match_val, match_len = self.trie.longest_match(chunk, idx)
|
|
@@ -1,33 +1,29 @@
|
|
|
1
1
|
import json
|
|
2
|
-
import logging
|
|
3
2
|
import os
|
|
4
3
|
|
|
5
4
|
from nepali_unicoder.trie import Trie
|
|
6
5
|
|
|
7
|
-
_logger = logging.getLogger(__name__)
|
|
8
|
-
|
|
9
6
|
|
|
10
7
|
def load_json_data(filename):
|
|
8
|
+
# Bundled data files are required; a missing one is a broken install, so let it raise.
|
|
11
9
|
path = os.path.join(os.path.dirname(__file__), "data", filename)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
return json.load(f)
|
|
15
|
-
except FileNotFoundError:
|
|
16
|
-
_logger.warning("Rule file %s not found.", filename)
|
|
17
|
-
return {}
|
|
10
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
11
|
+
return json.load(f)
|
|
18
12
|
|
|
19
13
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
14
|
+
def with_capitalized(mappings: dict) -> dict:
|
|
15
|
+
"""Add a capitalized variant of each key ('nepal' -> also 'Nepal')
|
|
16
|
+
unless that variant is listed explicitly."""
|
|
17
|
+
word_map = {roman.capitalize(): dev for roman, dev in mappings.items()}
|
|
18
|
+
word_map.update(mappings)
|
|
19
|
+
return word_map
|
|
20
|
+
|
|
25
21
|
|
|
22
|
+
class RuleLoader:
|
|
26
23
|
def load(self) -> Trie:
|
|
27
|
-
"""Load rules
|
|
24
|
+
"""Load character rules into a Trie."""
|
|
28
25
|
trie = Trie()
|
|
29
26
|
self._load_rules(trie)
|
|
30
|
-
self._load_custom_mappings(trie)
|
|
31
27
|
return trie
|
|
32
28
|
|
|
33
29
|
def _load_rules(self, trie: Trie):
|
|
@@ -86,21 +82,6 @@ class RuleLoader:
|
|
|
86
82
|
for rom, dev in digits.items():
|
|
87
83
|
trie.add(rom, dev)
|
|
88
84
|
|
|
89
|
-
def _load_custom_mappings(self, trie: Trie):
|
|
90
|
-
if not os.path.exists(self.word_maps_path):
|
|
91
|
-
return
|
|
92
|
-
|
|
93
|
-
with open(self.word_maps_path, "r", encoding="utf-8") as f:
|
|
94
|
-
mappings = json.load(f)
|
|
95
|
-
|
|
96
|
-
for roman, devanagari in mappings.items():
|
|
97
|
-
trie.add(roman, devanagari)
|
|
98
|
-
# Automatically add capitalized version if it doesn't exist
|
|
99
|
-
# (e.g. if 'nepal' is in maps, also add 'Nepal')
|
|
100
|
-
cap_roman = roman.capitalize()
|
|
101
|
-
if cap_roman not in mappings:
|
|
102
|
-
trie.add(cap_roman, devanagari)
|
|
103
|
-
|
|
104
85
|
|
|
105
86
|
class PreetiLoader:
|
|
106
87
|
def __init__(self):
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import re
|
|
2
4
|
from dataclasses import dataclass
|
|
3
|
-
from typing import List
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
@dataclass
|
|
@@ -14,7 +15,7 @@ class Tokenizer:
|
|
|
14
15
|
_RE_BLOCK = re.compile(r"\{([^}]*)\}")
|
|
15
16
|
_RE_NUMBER = re.compile(r"\d*\.\d+|\d+")
|
|
16
17
|
|
|
17
|
-
def tokenize(self, text: str
|
|
18
|
+
def tokenize(self, text: str) -> list[Token]:
|
|
18
19
|
"""
|
|
19
20
|
Split text into tokens using regex-based matching for robustness.
|
|
20
21
|
"""
|
|
@@ -39,43 +40,42 @@ class Tokenizer:
|
|
|
39
40
|
i += 3
|
|
40
41
|
continue
|
|
41
42
|
|
|
42
|
-
# 2. Check for Blocks and Braces
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
continue
|
|
43
|
+
# 2. Check for Blocks and Braces
|
|
44
|
+
# Escaped brace {{
|
|
45
|
+
if text.startswith("{{", i):
|
|
46
|
+
flush_roman()
|
|
47
|
+
tokens.append(Token(value="{", type="LITERAL"))
|
|
48
|
+
i += 2
|
|
49
|
+
continue
|
|
50
|
+
|
|
51
|
+
# Block {content}
|
|
52
|
+
match_block = re_block.match(text, i)
|
|
53
|
+
if match_block:
|
|
54
|
+
flush_roman()
|
|
55
|
+
tokens.append(Token(value=match_block.group(1), type="BLOCK"))
|
|
56
|
+
i = match_block.end()
|
|
57
|
+
continue
|
|
58
|
+
|
|
59
|
+
# Escaped brace }}
|
|
60
|
+
if text.startswith("}}", i):
|
|
61
|
+
flush_roman()
|
|
62
|
+
tokens.append(Token(value="}", type="LITERAL"))
|
|
63
|
+
i += 2
|
|
64
|
+
continue
|
|
65
|
+
|
|
66
|
+
# Unmatched closing brace }
|
|
67
|
+
if text.startswith("}", i):
|
|
68
|
+
flush_roman()
|
|
69
|
+
tokens.append(Token(value="}", type="LITERAL"))
|
|
70
|
+
i += 1
|
|
71
|
+
continue
|
|
72
|
+
|
|
73
|
+
# Unmatched opening brace {
|
|
74
|
+
if text.startswith("{", i):
|
|
75
|
+
flush_roman()
|
|
76
|
+
tokens.append(Token(value="{", type="LITERAL"))
|
|
77
|
+
i += 1
|
|
78
|
+
continue
|
|
79
79
|
|
|
80
80
|
# 3. Check for Numbers
|
|
81
81
|
match_number = re_number.match(text, i)
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
from
|
|
1
|
+
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class TrieNode:
|
|
5
|
-
__slots__ = ("children", "
|
|
5
|
+
__slots__ = ("children", "is_end", "value")
|
|
6
6
|
|
|
7
7
|
def __init__(self):
|
|
8
|
-
self.children:
|
|
9
|
-
self.value:
|
|
8
|
+
self.children: dict[str, TrieNode] = {}
|
|
9
|
+
self.value: str | None = None
|
|
10
10
|
self.is_end: bool = False
|
|
11
11
|
|
|
12
12
|
|
|
@@ -24,9 +24,7 @@ class Trie:
|
|
|
24
24
|
node.is_end = True
|
|
25
25
|
node.value = value
|
|
26
26
|
|
|
27
|
-
def longest_match(
|
|
28
|
-
self, text: str, start_index: int = 0
|
|
29
|
-
) -> Tuple[Optional[str], int]:
|
|
27
|
+
def longest_match(self, text: str, start_index: int = 0) -> tuple[str | None, int]:
|
|
30
28
|
"""
|
|
31
29
|
Find the longest key in the Trie that matches the start of the text.
|
|
32
30
|
Returns (value, length_of_match).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
|
-
Name:
|
|
3
|
-
Version: 0.
|
|
2
|
+
Name: nepali-unicoder
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A Python package for Roman to Nepali (Devanagari) transliteration
|
|
5
5
|
Author-email: realsanjeev <075bei033.sanjeev@pcampus.edu.np>
|
|
6
6
|
License: MIT
|
|
@@ -11,7 +11,7 @@ Keywords: nepali,unicoder,roman to devanagari,nepali transliteration,preeti to u
|
|
|
11
11
|
Classifier: Programming Language :: Python
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
13
|
Classifier: Operating System :: OS Independent
|
|
14
|
-
Requires-Python: >=3.
|
|
14
|
+
Requires-Python: >=3.8
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
16
|
License-File: LICENSE
|
|
17
17
|
Provides-Extra: dev
|
|
@@ -36,7 +36,7 @@ Read the **[Full Documentation](https://realsanjeev.github.io/nepali_unicoder/)*
|
|
|
36
36
|
- **Smart Vowel Handling**: Distinguishes between independent vowels (e.g., 'aa' -> 'आ') and vowel signs/matras (e.g., 'ka' -> 'क', 'kaa' -> 'का').
|
|
37
37
|
- **Contextual Rules**: Handles complex Devanagari rules like reph positioning, matra reordering, and special character combinations.
|
|
38
38
|
- **Mixed Content Support**: Allows keeping English words or specific text in Roman script using `{}` blocks.
|
|
39
|
-
- **Customizable**: Supports custom word-level overrides via `word_maps.json`.
|
|
39
|
+
- **Customizable**: Supports custom word-level overrides via `word_maps.json` or `Converter(word_map={...})`.
|
|
40
40
|
- **CLI Support**: Can be used directly from the command line.
|
|
41
41
|
|
|
42
42
|
## Installation
|
|
@@ -83,6 +83,11 @@ print(converter.convert(text))
|
|
|
83
83
|
mixed_text = "mero naam {Sanjeev} ho"
|
|
84
84
|
print(converter.convert(mixed_text))
|
|
85
85
|
# Output: मेरो नाम् Sanjeev हो
|
|
86
|
+
|
|
87
|
+
# Your own whole-word mappings (override the bundled ones)
|
|
88
|
+
converter = Converter(word_map={"sanjeev": "सञ्जीव"})
|
|
89
|
+
print(converter.convert("Sanjeev"))
|
|
90
|
+
# Output: सञ्जीव
|
|
86
91
|
```
|
|
87
92
|
|
|
88
93
|
### Preeti Mode
|
|
@@ -7,7 +7,6 @@ src/nepali_unicoder/__main__.py
|
|
|
7
7
|
src/nepali_unicoder/convert.py
|
|
8
8
|
src/nepali_unicoder/engine.py
|
|
9
9
|
src/nepali_unicoder/loader.py
|
|
10
|
-
src/nepali_unicoder/rules.py
|
|
11
10
|
src/nepali_unicoder/tokenizer.py
|
|
12
11
|
src/nepali_unicoder/trie.py
|
|
13
12
|
src/nepali_unicoder.egg-info/PKG-INFO
|
|
@@ -125,6 +125,7 @@ class TestPublicAPI(unittest.TestCase):
|
|
|
125
125
|
# Import at module level already asserts this; just confirm it's the
|
|
126
126
|
# same class as the one in convert.py
|
|
127
127
|
from nepali_unicoder.convert import Converter as _Converter
|
|
128
|
+
|
|
128
129
|
self.assertIs(Converter, _Converter)
|
|
129
130
|
|
|
130
131
|
def test_basic_conversion_via_public_api(self):
|
|
@@ -136,6 +137,32 @@ class TestPublicAPI(unittest.TestCase):
|
|
|
136
137
|
self.assertEqual(c.convert("s{"), "र्क")
|
|
137
138
|
|
|
138
139
|
|
|
140
|
+
class TestCustomWordMap(unittest.TestCase):
|
|
141
|
+
def test_user_words(self):
|
|
142
|
+
c = Converter(word_map={"sanjeev": "सञ्जीव", "bus": "बस्"})
|
|
143
|
+
self.assertEqual(c.convert("sanjeev"), "सञ्जीव")
|
|
144
|
+
self.assertEqual(c.convert("Sanjeev"), "सञ्जीव") # capitalized variant
|
|
145
|
+
self.assertEqual(c.convert("bus"), "बस्") # overrides bundled entry
|
|
146
|
+
self.assertEqual(c.convert("nepal"), "नेपाल") # bundled entries kept
|
|
147
|
+
self.assertEqual(c.convert("sanjeevko"), "सन्जीव्को") # whole words only
|
|
148
|
+
|
|
149
|
+
def test_user_words_do_not_leak(self):
|
|
150
|
+
Converter(word_map={"bus": "बस्"})
|
|
151
|
+
self.assertEqual(Converter().convert("bus"), "बस")
|
|
152
|
+
|
|
153
|
+
def test_invalid_word_map(self):
|
|
154
|
+
with self.assertRaises(ValueError):
|
|
155
|
+
Converter(word_map={"two words": "x"})
|
|
156
|
+
with self.assertRaises(ValueError):
|
|
157
|
+
Converter(mode="preeti", word_map={"abc": "x"})
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class TestTableCache(unittest.TestCase):
|
|
161
|
+
def test_tables_shared_per_mode(self):
|
|
162
|
+
self.assertIs(Converter().engine.trie, Converter().engine.trie)
|
|
163
|
+
self.assertIs(Converter("preeti").engine.trie, Converter("preeti").engine.trie)
|
|
164
|
+
self.assertIsNot(Converter().engine.trie, Converter("preeti").engine.trie)
|
|
165
|
+
|
|
139
166
|
|
|
140
167
|
if __name__ == "__main__":
|
|
141
168
|
unittest.main()
|
|
@@ -104,7 +104,7 @@ class TestNepaliUnicoder(unittest.TestCase):
|
|
|
104
104
|
self.assertEqual(self.converter.convert("lRi"), "ऌ")
|
|
105
105
|
# Consonant l + Ri matra is still accessible via 'l' alone + separate vowel
|
|
106
106
|
self.assertEqual(self.converter.convert("la"), "ल")
|
|
107
|
-
self.assertEqual(self.converter.convert("lRi"), "ऌ")
|
|
107
|
+
self.assertEqual(self.converter.convert("lRi"), "ऌ") # vowel wins
|
|
108
108
|
|
|
109
109
|
def test_word_maps_case_sensitivity(self):
|
|
110
110
|
"""Fix #5: word_maps keys are now case-sensitive; both uppercase and
|
|
@@ -117,11 +117,23 @@ class TestNepaliUnicoder(unittest.TestCase):
|
|
|
117
117
|
self.assertEqual(self.converter.convert("Kathmandu"), "काठमाडौं")
|
|
118
118
|
self.assertEqual(self.converter.convert("SMS"), "एसएमएस")
|
|
119
119
|
|
|
120
|
+
def test_word_maps_whole_words_only(self):
|
|
121
|
+
"""Word maps must not fire inside longer words ('bus' in 'bushan')."""
|
|
122
|
+
self.assertEqual(self.converter.convert("bus"), "बस")
|
|
123
|
+
self.assertEqual(self.converter.convert("bus chadhe"), "बस चधे")
|
|
124
|
+
self.assertEqual(self.converter.convert("bushan"), "बुशन्")
|
|
125
|
+
self.assertEqual(self.converter.convert("plant"), "प्लन्त्")
|
|
126
|
+
self.assertEqual(self.converter.convert("nepal, bus."), "नेपाल, बस।")
|
|
127
|
+
|
|
120
128
|
|
|
121
129
|
class TestPreetiUnicoder(unittest.TestCase):
|
|
122
130
|
def setUp(self):
|
|
123
131
|
self.converter = Converter(mode="preeti")
|
|
124
132
|
|
|
133
|
+
def test_digits_and_dots_use_preeti_map(self):
|
|
134
|
+
"""Digits and '.' are Preeti glyphs, not a decimal number."""
|
|
135
|
+
self.assertEqual(self.converter.convert("1.5"), "ज्ञ।छ")
|
|
136
|
+
|
|
125
137
|
def test_qwerty_alphabet(self):
|
|
126
138
|
"""Basic character mappings (a-z, A-Z)"""
|
|
127
139
|
self.assertEqual(
|
|
@@ -223,6 +235,9 @@ class TestPreetiUnicoder(unittest.TestCase):
|
|
|
223
235
|
self.assertEqual(self.converter.convert(";ËLtsf]"), "सङ्गीतको")
|
|
224
236
|
self.assertEqual(self.converter.convert("5'§}"), "छुट्टै")
|
|
225
237
|
self.assertEqual(self.converter.convert("5'§f5'§}"), "छुट्टाछुट्टै")
|
|
238
|
+
self.assertEqual(self.converter.convert("s[lqd al'4dQf"), "कृत्रिम बुद्धिमत्ता")
|
|
239
|
+
self.assertEqual(self.converter.convert("cfl6{lkmlzon"), "आर्टिफिशियल")
|
|
240
|
+
self.assertEqual(self.converter.convert(";f/f+z M"), "सारांश:")
|
|
226
241
|
|
|
227
242
|
def test_duplicate_removal(self):
|
|
228
243
|
"""Test that duplicate matras are removed"""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{nepali_unicoder-0.2.0 → nepali_unicoder-0.3.0}/src/nepali_unicoder.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|