nepali_unicoder 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nepali_unicoder/__init__.py +0 -0
- nepali_unicoder/__main__.py +32 -0
- nepali_unicoder/convert.py +13 -0
- nepali_unicoder/engine.py +97 -0
- nepali_unicoder/loader.py +84 -0
- nepali_unicoder/rules.py +262 -0
- nepali_unicoder/tokenizer.py +69 -0
- nepali_unicoder/trie.py +47 -0
- nepali_unicoder/word_maps.json +67 -0
- nepali_unicoder-0.1.1.dist-info/METADATA +183 -0
- nepali_unicoder-0.1.1.dist-info/RECORD +14 -0
- nepali_unicoder-0.1.1.dist-info/WHEEL +5 -0
- nepali_unicoder-0.1.1.dist-info/licenses/LICENSE +21 -0
- nepali_unicoder-0.1.1.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
from nepali_unicoder.convert import Converter
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def main():
|
|
7
|
+
args = sys.argv[1:]
|
|
8
|
+
mode = "roman"
|
|
9
|
+
|
|
10
|
+
if "--preeti" in args:
|
|
11
|
+
mode = "preeti"
|
|
12
|
+
args.remove("--preeti")
|
|
13
|
+
|
|
14
|
+
if len(args) < 1:
|
|
15
|
+
# Check if there is stdin
|
|
16
|
+
if not sys.stdin.isatty():
|
|
17
|
+
input_text = sys.stdin.read()
|
|
18
|
+
else:
|
|
19
|
+
print('Usage: python -m nepali_unicoder "your roman text here"')
|
|
20
|
+
print(' python -m nepali_unicoder --preeti "your preeti text here"')
|
|
21
|
+
print(' or: echo "namaste" | python -m nepali_unicoder')
|
|
22
|
+
return
|
|
23
|
+
else:
|
|
24
|
+
input_text = " ".join(args)
|
|
25
|
+
|
|
26
|
+
converter = Converter(mode=mode)
|
|
27
|
+
result = converter.convert(input_text)
|
|
28
|
+
print(result)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
if __name__ == "__main__":
|
|
32
|
+
main()
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from nepali_unicoder.engine import Engine
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Converter:
|
|
5
|
+
"""
|
|
6
|
+
Wrapper around Engine for backward compatibility.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
def __init__(self, mode: str = "roman"):
|
|
10
|
+
self.engine = Engine(mode=mode)
|
|
11
|
+
|
|
12
|
+
def convert(self, text: str) -> str:
|
|
13
|
+
return self.engine.transliterate(text)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from nepali_unicoder.loader import PreetiLoader, RuleLoader
|
|
5
|
+
from nepali_unicoder.rules import PREETI_TO_UNICODE_MAPPING
|
|
6
|
+
from nepali_unicoder.tokenizer import Tokenizer
|
|
7
|
+
from nepali_unicoder.trie import Trie
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Engine:
|
|
11
|
+
def __init__(
|
|
12
|
+
self,
|
|
13
|
+
trie: Optional[Trie] = None,
|
|
14
|
+
tokenizer: Optional[Tokenizer] = None,
|
|
15
|
+
mode: str = "roman",
|
|
16
|
+
):
|
|
17
|
+
self.mode = mode
|
|
18
|
+
self.post_rules = []
|
|
19
|
+
|
|
20
|
+
if trie is None:
|
|
21
|
+
if mode == "preeti":
|
|
22
|
+
loader = PreetiLoader()
|
|
23
|
+
# Load post-processing rules for Preeti mode
|
|
24
|
+
self.post_rules = PREETI_TO_UNICODE_MAPPING.get("post_rules", [])
|
|
25
|
+
else:
|
|
26
|
+
loader = RuleLoader()
|
|
27
|
+
self.trie = loader.load()
|
|
28
|
+
else:
|
|
29
|
+
self.trie = trie
|
|
30
|
+
|
|
31
|
+
if tokenizer is None:
|
|
32
|
+
self.tokenizer = Tokenizer()
|
|
33
|
+
else:
|
|
34
|
+
self.tokenizer = tokenizer
|
|
35
|
+
|
|
36
|
+
def transliterate(self, text: str) -> str:
|
|
37
|
+
"""
|
|
38
|
+
Convert Roman text to Devanagari using the Trie.
|
|
39
|
+
"""
|
|
40
|
+
if not text:
|
|
41
|
+
return ""
|
|
42
|
+
|
|
43
|
+
tokens = self.tokenizer.tokenize(text)
|
|
44
|
+
result = []
|
|
45
|
+
|
|
46
|
+
for token in tokens:
|
|
47
|
+
if token.type == "BLOCK":
|
|
48
|
+
result.append(token.value)
|
|
49
|
+
elif token.type == "LITERAL":
|
|
50
|
+
result.append(token.value)
|
|
51
|
+
elif token.type == "NUMBER":
|
|
52
|
+
# Process Number chunk: transliterate digits, keep others (like .) as is
|
|
53
|
+
for char in token.value:
|
|
54
|
+
if char.isdigit():
|
|
55
|
+
# Use Trie to find digit mapping (digits are single chars in rules)
|
|
56
|
+
match_val, _ = self.trie.longest_match(char)
|
|
57
|
+
if match_val:
|
|
58
|
+
result.append(match_val)
|
|
59
|
+
else:
|
|
60
|
+
result.append(char)
|
|
61
|
+
else:
|
|
62
|
+
result.append(char)
|
|
63
|
+
elif token.type == "ROMAN":
|
|
64
|
+
# Process Roman chunk with Trie
|
|
65
|
+
chunk = token.value
|
|
66
|
+
idx = 0
|
|
67
|
+
chunk_len = len(chunk)
|
|
68
|
+
|
|
69
|
+
while idx < chunk_len:
|
|
70
|
+
# Try to find longest match starting at idx
|
|
71
|
+
sub = chunk[idx:]
|
|
72
|
+
match_val, match_len = self.trie.longest_match(sub)
|
|
73
|
+
|
|
74
|
+
if match_val:
|
|
75
|
+
result.append(match_val)
|
|
76
|
+
idx += match_len
|
|
77
|
+
else:
|
|
78
|
+
# No match, keep character as is
|
|
79
|
+
result.append(chunk[idx])
|
|
80
|
+
idx += 1
|
|
81
|
+
|
|
82
|
+
output = "".join(result)
|
|
83
|
+
|
|
84
|
+
# Apply post-processing rules for Preeti mode
|
|
85
|
+
if self.mode == "preeti" and self.post_rules:
|
|
86
|
+
output = self._apply_post_rules(output)
|
|
87
|
+
|
|
88
|
+
return output
|
|
89
|
+
|
|
90
|
+
def _apply_post_rules(self, text: str) -> str:
|
|
91
|
+
"""
|
|
92
|
+
Apply post-processing rules (regex replacements) to the text.
|
|
93
|
+
Used for Preeti mode to handle contextual transformations.
|
|
94
|
+
"""
|
|
95
|
+
for pattern, replacement in self.post_rules:
|
|
96
|
+
text = re.sub(pattern, replacement, text)
|
|
97
|
+
return text
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from nepali_unicoder.rules import PREETI_TO_UNICODE_MAPPING, ROMAN_TO_UNICODE_RULES
|
|
5
|
+
from nepali_unicoder.trie import Trie
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class RuleLoader:
|
|
9
|
+
def __init__(self):
|
|
10
|
+
self.word_maps_path = os.path.join(os.path.dirname(__file__), "word_maps.json")
|
|
11
|
+
|
|
12
|
+
def load(self) -> Trie:
|
|
13
|
+
"""Load rules and custom mappings into a Trie."""
|
|
14
|
+
trie = Trie()
|
|
15
|
+
self._load_rules(trie)
|
|
16
|
+
self._load_custom_mappings(trie)
|
|
17
|
+
return trie
|
|
18
|
+
|
|
19
|
+
def _load_rules(self, trie: Trie):
|
|
20
|
+
data = ROMAN_TO_UNICODE_RULES
|
|
21
|
+
|
|
22
|
+
consonants = data.get("consonants", {})
|
|
23
|
+
vowels = data.get("vowels", {})
|
|
24
|
+
matras = data.get("matras", {})
|
|
25
|
+
special = data.get("special", {})
|
|
26
|
+
digits = data.get("digits", {})
|
|
27
|
+
|
|
28
|
+
# 1. Independent Vowels
|
|
29
|
+
for rom, dev in vowels.items():
|
|
30
|
+
trie.add(rom, dev)
|
|
31
|
+
|
|
32
|
+
# 2. Consonants and Combinations
|
|
33
|
+
halanta = "्"
|
|
34
|
+
|
|
35
|
+
for rom_cons, dev_cons in consonants.items():
|
|
36
|
+
# Case 1: Consonant alone (halanta form) -> 'k' -> 'क्'
|
|
37
|
+
trie.add(rom_cons, dev_cons + halanta)
|
|
38
|
+
|
|
39
|
+
# Case 2: Consonant + 'a' (Schwa form) -> 'ka' -> 'क'
|
|
40
|
+
trie.add(rom_cons + "a", dev_cons)
|
|
41
|
+
|
|
42
|
+
# Case 3: Consonant + other vowels -> 'ki' -> 'कि'
|
|
43
|
+
for rom_vowel, matra in matras.items():
|
|
44
|
+
if rom_vowel == "a":
|
|
45
|
+
continue # Handled above
|
|
46
|
+
trie.add(rom_cons + rom_vowel, dev_cons + matra)
|
|
47
|
+
|
|
48
|
+
# 3. Special, Digits, Punctuation
|
|
49
|
+
for rom, dev in special.items():
|
|
50
|
+
trie.add(rom, dev)
|
|
51
|
+
|
|
52
|
+
for rom, dev in digits.items():
|
|
53
|
+
trie.add(rom, dev)
|
|
54
|
+
|
|
55
|
+
# Ensure 'a' maps to 'अ' (already in vowels, but good to double check)
|
|
56
|
+
trie.add("a", vowels.get("a", "अ"))
|
|
57
|
+
|
|
58
|
+
def _load_custom_mappings(self, trie: Trie):
|
|
59
|
+
if not os.path.exists(self.word_maps_path):
|
|
60
|
+
return
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
with open(self.word_maps_path, "r", encoding="utf-8") as f:
|
|
64
|
+
mappings = json.load(f)
|
|
65
|
+
for roman, devanagari in mappings.items():
|
|
66
|
+
trie.add(roman.lower(), devanagari)
|
|
67
|
+
except Exception as e:
|
|
68
|
+
print(f"Error reading word_maps.json: {e}")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class PreetiLoader:
|
|
72
|
+
def __init__(self):
|
|
73
|
+
pass
|
|
74
|
+
|
|
75
|
+
def load(self) -> Trie:
|
|
76
|
+
"""Load Preeti rules into a Trie."""
|
|
77
|
+
trie = Trie()
|
|
78
|
+
data = PREETI_TO_UNICODE_MAPPING
|
|
79
|
+
|
|
80
|
+
mappings = data.get("mappings", {})
|
|
81
|
+
for key, value in mappings.items():
|
|
82
|
+
trie.add(key, value)
|
|
83
|
+
|
|
84
|
+
return trie
|
nepali_unicoder/rules.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
PREETI_TO_UNICODE_MAPPING = {
|
|
2
|
+
"mappings": {
|
|
3
|
+
# Lowercase letters (a-z)
|
|
4
|
+
"a": "ब",
|
|
5
|
+
"b": "द",
|
|
6
|
+
"c": "अ",
|
|
7
|
+
"d": "म",
|
|
8
|
+
"e": "भ",
|
|
9
|
+
"f": "ा",
|
|
10
|
+
"g": "न",
|
|
11
|
+
"h": "ज",
|
|
12
|
+
"i": "ष्",
|
|
13
|
+
"j": "व",
|
|
14
|
+
"k": "प",
|
|
15
|
+
"l": "ि",
|
|
16
|
+
"m": "m",
|
|
17
|
+
"n": "ल",
|
|
18
|
+
"o": "य",
|
|
19
|
+
"p": "उ",
|
|
20
|
+
"q": "त्र",
|
|
21
|
+
"r": "च",
|
|
22
|
+
"s": "क",
|
|
23
|
+
"t": "त",
|
|
24
|
+
"u": "ग",
|
|
25
|
+
"v": "ख",
|
|
26
|
+
"w": "ध",
|
|
27
|
+
"x": "ह",
|
|
28
|
+
"y": "थ",
|
|
29
|
+
"z": "श",
|
|
30
|
+
# Uppercase letters (A-Z)
|
|
31
|
+
"A": "ब्",
|
|
32
|
+
"B": "द्य",
|
|
33
|
+
"C": "ऋ",
|
|
34
|
+
"D": "म्",
|
|
35
|
+
"E": "भ्",
|
|
36
|
+
"F": "ँ",
|
|
37
|
+
"G": "न्",
|
|
38
|
+
"H": "ज्",
|
|
39
|
+
"I": "क्ष्",
|
|
40
|
+
"J": "व्",
|
|
41
|
+
"K": "प्",
|
|
42
|
+
"L": "ी",
|
|
43
|
+
"M": "ः",
|
|
44
|
+
"N": "ल्",
|
|
45
|
+
"O": "इ",
|
|
46
|
+
"P": "ए",
|
|
47
|
+
"Q": "त्त",
|
|
48
|
+
"R": "च्",
|
|
49
|
+
"S": "क्",
|
|
50
|
+
"T": "त्",
|
|
51
|
+
"U": "ग्",
|
|
52
|
+
"V": "ख्",
|
|
53
|
+
"W": "ध्",
|
|
54
|
+
"X": "ह्",
|
|
55
|
+
"Y": "थ्",
|
|
56
|
+
"Z": "श्",
|
|
57
|
+
# Numbers (0-9)
|
|
58
|
+
"0": "ण्",
|
|
59
|
+
"1": "ज्ञ",
|
|
60
|
+
"2": "द्द",
|
|
61
|
+
"3": "घ",
|
|
62
|
+
"4": "द्ध",
|
|
63
|
+
"5": "छ",
|
|
64
|
+
"6": "ट",
|
|
65
|
+
"7": "ठ",
|
|
66
|
+
"8": "ड",
|
|
67
|
+
"9": "ढ",
|
|
68
|
+
# Special characters (Shift + numbers)
|
|
69
|
+
"!": "१",
|
|
70
|
+
"@": "२",
|
|
71
|
+
"#": "३",
|
|
72
|
+
"$": "४",
|
|
73
|
+
"%": "५",
|
|
74
|
+
"^": "६",
|
|
75
|
+
"&": "७",
|
|
76
|
+
"*": "८",
|
|
77
|
+
"(": "९",
|
|
78
|
+
")": "०",
|
|
79
|
+
# Brackets and punctuation
|
|
80
|
+
"[": "ृ",
|
|
81
|
+
"]": "े",
|
|
82
|
+
"{": "{",
|
|
83
|
+
"}": "ै",
|
|
84
|
+
"|": "्र",
|
|
85
|
+
# Other special characters
|
|
86
|
+
";": "स",
|
|
87
|
+
":": "स्",
|
|
88
|
+
"'": "ु",
|
|
89
|
+
'"': "ू",
|
|
90
|
+
",": ",",
|
|
91
|
+
".": "।",
|
|
92
|
+
"/": "र",
|
|
93
|
+
"<": "?",
|
|
94
|
+
">": "श्र",
|
|
95
|
+
"?": "रु",
|
|
96
|
+
"\\": "्",
|
|
97
|
+
"`": "ञ",
|
|
98
|
+
"~": "ञ्",
|
|
99
|
+
"_": ")",
|
|
100
|
+
"-": "(",
|
|
101
|
+
"=": ".",
|
|
102
|
+
"+": "ं",
|
|
103
|
+
},
|
|
104
|
+
# Post-processing rules applied after character mapping
|
|
105
|
+
# Format: [pattern, replacement] - applied in order using regex
|
|
106
|
+
"post_rules": [
|
|
107
|
+
# 1. Remove invalid combinations
|
|
108
|
+
["्ा", ""], # halant + aa-kar is invalid
|
|
109
|
+
# 2. Handle 'm' character contextually
|
|
110
|
+
[
|
|
111
|
+
r"(त्र|त्त)([^उभप]+?)m",
|
|
112
|
+
r"\1m\2",
|
|
113
|
+
], # preserve m position for certain combinations
|
|
114
|
+
["त्रm", "क्र"], # त्र + m = क्र
|
|
115
|
+
["त्तm", "क्त"], # त्त + m = क्त
|
|
116
|
+
[r"([^उभप]+?)m", r"m\1"], # move m before (except after उ, भ, प)
|
|
117
|
+
["उm", "ऊ"], # उ + m = ऊ
|
|
118
|
+
["भm", "झ"], # भ + m = झ
|
|
119
|
+
["पm", "फ"], # प + m = फ
|
|
120
|
+
# 3. Handle reph ({) contextually
|
|
121
|
+
["इ{", "ई"], # इ + { = ई (special case)
|
|
122
|
+
[r"ि((.्)*[^्])", r"\1ि"], # move short i (ि) after consonant
|
|
123
|
+
[r"(.[ािीुूृेैोौंःँ]*?){", r"{\1"], # move { before matras
|
|
124
|
+
[r"((.्)*){", r"{\1"], # move { before halant sequences
|
|
125
|
+
["{", "र्"], # finally convert { to reph (र्)
|
|
126
|
+
# 4. Reorder matras (vowel signs) to correct positions
|
|
127
|
+
[r"([ाीुूृेैोौंःँ]+?)(्(.्)*[^्])", r"\2\1"], # move matras after halant+consonant
|
|
128
|
+
[r"्([ाीुूृेैोौंःँ]+?)((.्)*[^्])", r"्\2\1"], # reorder matras after halant
|
|
129
|
+
# 5. Move anusvara and chandrabindu to end
|
|
130
|
+
[r"([ंँ])([ािीुूृेैोौः]*)", r"\2\1"], # move ं/ँ after other matras
|
|
131
|
+
# 6. Remove duplicates
|
|
132
|
+
["ँँ", "ँ"], # duplicate chandrabindu
|
|
133
|
+
["ंं", "ं"], # duplicate anusvara
|
|
134
|
+
["ेे", "े"], # duplicate e-matra
|
|
135
|
+
["ैै", "ै"], # duplicate ai-matra
|
|
136
|
+
["ुु", "ु"], # duplicate u-matra
|
|
137
|
+
["ूू", "ू"], # duplicate uu-matra
|
|
138
|
+
# 7. Handle visarga at start of line
|
|
139
|
+
["^ः", ":"], # visarga at start should remain colon
|
|
140
|
+
# 8. Special combinations
|
|
141
|
+
["टृ", "ट्ट"], # ट + ृ = ट्ट
|
|
142
|
+
# 9. Combine vowel signs with independent vowels
|
|
143
|
+
["ेा", "ाे"], # reorder e-matra and aa-matra
|
|
144
|
+
["ैा", "ाै"], # reorder ai-matra and aa-matra
|
|
145
|
+
["अाे", "ओ"], # अ + ा + े = ओ
|
|
146
|
+
["अाै", "औ"], # अ + ा + ै = औ
|
|
147
|
+
["अा", "आ"], # अ + ा = आ
|
|
148
|
+
["एे", "ऐ"], # ए + े = ऐ
|
|
149
|
+
["ाे", "ो"], # ा + े = ो
|
|
150
|
+
["ाै", "ौ"], # ा + ै = ौ
|
|
151
|
+
],
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
ROMAN_TO_UNICODE_RULES = {
|
|
156
|
+
"consonants": {
|
|
157
|
+
"k": "क",
|
|
158
|
+
"kh": "ख",
|
|
159
|
+
"g": "ग",
|
|
160
|
+
"gh": "घ",
|
|
161
|
+
"ng": "ङ",
|
|
162
|
+
"ch": "च",
|
|
163
|
+
"chh": "छ",
|
|
164
|
+
"j": "ज",
|
|
165
|
+
"jh": "झ",
|
|
166
|
+
"ny": "ञ",
|
|
167
|
+
"T": "ट",
|
|
168
|
+
"Th": "ठ",
|
|
169
|
+
"D": "ड",
|
|
170
|
+
"Dh": "ढ",
|
|
171
|
+
"N": "ण",
|
|
172
|
+
"t": "त",
|
|
173
|
+
"th": "थ",
|
|
174
|
+
"d": "द",
|
|
175
|
+
"dh": "ध",
|
|
176
|
+
"n": "न",
|
|
177
|
+
"p": "प",
|
|
178
|
+
"ph": "फ",
|
|
179
|
+
"f": "फ",
|
|
180
|
+
"b": "ब",
|
|
181
|
+
"bh": "भ",
|
|
182
|
+
"m": "म",
|
|
183
|
+
"y": "य",
|
|
184
|
+
"r": "र",
|
|
185
|
+
"l": "ल",
|
|
186
|
+
"v": "व",
|
|
187
|
+
"w": "व",
|
|
188
|
+
"sh": "श",
|
|
189
|
+
"Sh": "ष",
|
|
190
|
+
"s": "स",
|
|
191
|
+
"h": "ह",
|
|
192
|
+
"ksh": "क्ष",
|
|
193
|
+
"tra": "त्र",
|
|
194
|
+
"gy": "ज्ञ",
|
|
195
|
+
"shr": "श्र",
|
|
196
|
+
},
|
|
197
|
+
"vowels": {
|
|
198
|
+
"a": "अ",
|
|
199
|
+
"aa": "आ",
|
|
200
|
+
"A": "आ",
|
|
201
|
+
"i": "इ",
|
|
202
|
+
"I": "इ",
|
|
203
|
+
"ee": "ई",
|
|
204
|
+
"ii": "ई",
|
|
205
|
+
"u": "उ",
|
|
206
|
+
"U": "उ",
|
|
207
|
+
"oo": "ऊ",
|
|
208
|
+
"uu": "ऊ",
|
|
209
|
+
"Ri": "ऋ",
|
|
210
|
+
"R": "ऋ",
|
|
211
|
+
"Ree": "ॠ",
|
|
212
|
+
"lRi": "ऌ",
|
|
213
|
+
"e": "ए",
|
|
214
|
+
"ai": "ऐ",
|
|
215
|
+
"o": "ओ",
|
|
216
|
+
"au": "औ",
|
|
217
|
+
},
|
|
218
|
+
"matras": {
|
|
219
|
+
"a": "",
|
|
220
|
+
"aa": "ा",
|
|
221
|
+
"A": "ा",
|
|
222
|
+
"i": "ि",
|
|
223
|
+
"I": "ि",
|
|
224
|
+
"ee": "ी",
|
|
225
|
+
"ii": "ी",
|
|
226
|
+
"u": "ु",
|
|
227
|
+
"U": "ु",
|
|
228
|
+
"oo": "ू",
|
|
229
|
+
"uu": "ू",
|
|
230
|
+
"Ri": "ृ",
|
|
231
|
+
"R": "ृ",
|
|
232
|
+
"Ree": "ॄ",
|
|
233
|
+
"e": "े",
|
|
234
|
+
"ai": "ै",
|
|
235
|
+
"o": "ो",
|
|
236
|
+
"au": "ौ",
|
|
237
|
+
},
|
|
238
|
+
"special": {
|
|
239
|
+
"om": "ॐ",
|
|
240
|
+
"Om": "ॐ",
|
|
241
|
+
"aum": "ॐ",
|
|
242
|
+
".": "।",
|
|
243
|
+
"..": "॥",
|
|
244
|
+
"|": "।",
|
|
245
|
+
"||": "॥",
|
|
246
|
+
"M": "ं",
|
|
247
|
+
"H": "ः",
|
|
248
|
+
"~": "ँ",
|
|
249
|
+
},
|
|
250
|
+
"digits": {
|
|
251
|
+
"0": "०",
|
|
252
|
+
"1": "१",
|
|
253
|
+
"2": "२",
|
|
254
|
+
"3": "३",
|
|
255
|
+
"4": "४",
|
|
256
|
+
"5": "५",
|
|
257
|
+
"6": "६",
|
|
258
|
+
"7": "७",
|
|
259
|
+
"8": "८",
|
|
260
|
+
"9": "९",
|
|
261
|
+
},
|
|
262
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import List
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass
|
|
6
|
+
class Token:
|
|
7
|
+
value: str
|
|
8
|
+
type: str # 'ROMAN', 'BLOCK', 'PUNCTUATION', 'UNKNOWN', 'LITERAL', 'NUMBER'
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Tokenizer:
|
|
12
|
+
def tokenize(self, text: str) -> List[Token]:
|
|
13
|
+
"""
|
|
14
|
+
Split text into tokens.
|
|
15
|
+
This simple tokenizer handles {blocks}, numbers, and treats everything else as ROMAN for now.
|
|
16
|
+
"""
|
|
17
|
+
tokens = []
|
|
18
|
+
i = 0
|
|
19
|
+
n = len(text)
|
|
20
|
+
|
|
21
|
+
while i < n:
|
|
22
|
+
if text[i] == "{":
|
|
23
|
+
# Check for escape {{
|
|
24
|
+
if i + 1 < n and text[i + 1] == "{":
|
|
25
|
+
# Actually, if it's escaped, it should probably be treated as a literal '{' in the output.
|
|
26
|
+
tokens.append(Token(value="{", type="LITERAL"))
|
|
27
|
+
i += 2
|
|
28
|
+
else:
|
|
29
|
+
# Start of block
|
|
30
|
+
j = i + 1
|
|
31
|
+
while j < n and text[j] != "}":
|
|
32
|
+
j += 1
|
|
33
|
+
|
|
34
|
+
if j < n: # Found closing brace
|
|
35
|
+
content = text[i + 1 : j]
|
|
36
|
+
tokens.append(Token(value=content, type="BLOCK"))
|
|
37
|
+
i = j + 1
|
|
38
|
+
else: # Unclosed brace, treat as literal
|
|
39
|
+
tokens.append(Token(value="{", type="LITERAL"))
|
|
40
|
+
i += 1
|
|
41
|
+
elif text[i] == "}":
|
|
42
|
+
# Unmatched closing brace, treat as literal
|
|
43
|
+
tokens.append(Token(value="}", type="LITERAL"))
|
|
44
|
+
i += 1
|
|
45
|
+
elif text[i].isdigit():
|
|
46
|
+
# Start of a number
|
|
47
|
+
j = i
|
|
48
|
+
while j < n and text[j].isdigit():
|
|
49
|
+
j += 1
|
|
50
|
+
|
|
51
|
+
# Check for decimal part
|
|
52
|
+
if j < n and text[j] == ".":
|
|
53
|
+
# Look ahead to see if there are digits after dot
|
|
54
|
+
if j + 1 < n and text[j + 1].isdigit():
|
|
55
|
+
j += 1 # Consume dot
|
|
56
|
+
while j < n and text[j].isdigit():
|
|
57
|
+
j += 1
|
|
58
|
+
|
|
59
|
+
tokens.append(Token(value=text[i:j], type="NUMBER"))
|
|
60
|
+
i = j
|
|
61
|
+
else:
|
|
62
|
+
# Accumulate Roman text
|
|
63
|
+
j = i
|
|
64
|
+
while j < n and text[j] not in "{}" and not text[j].isdigit():
|
|
65
|
+
j += 1
|
|
66
|
+
tokens.append(Token(value=text[i:j], type="ROMAN"))
|
|
67
|
+
i = j
|
|
68
|
+
|
|
69
|
+
return tokens
|
nepali_unicoder/trie.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from typing import Dict, Optional, Tuple
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class TrieNode:
|
|
5
|
+
__slots__ = ("children", "value", "is_end")
|
|
6
|
+
|
|
7
|
+
def __init__(self):
|
|
8
|
+
self.children: Dict[str, TrieNode] = {}
|
|
9
|
+
self.value: Optional[str] = None
|
|
10
|
+
self.is_end: bool = False
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Trie:
|
|
14
|
+
def __init__(self):
|
|
15
|
+
self.root = TrieNode()
|
|
16
|
+
|
|
17
|
+
def add(self, key: str, value: str) -> None:
|
|
18
|
+
"""Insert a key-value pair into the Trie."""
|
|
19
|
+
node = self.root
|
|
20
|
+
for char in key:
|
|
21
|
+
if char not in node.children:
|
|
22
|
+
node.children[char] = TrieNode()
|
|
23
|
+
node = node.children[char]
|
|
24
|
+
node.is_end = True
|
|
25
|
+
node.value = value
|
|
26
|
+
|
|
27
|
+
def longest_match(self, text: str) -> Tuple[Optional[str], int]:
|
|
28
|
+
"""
|
|
29
|
+
Find the longest key in the Trie that matches the start of the text.
|
|
30
|
+
Returns (value, length_of_match).
|
|
31
|
+
If no match found, returns (None, 0).
|
|
32
|
+
"""
|
|
33
|
+
node = self.root
|
|
34
|
+
last_match_value = None
|
|
35
|
+
last_match_len = 0
|
|
36
|
+
current_len = 0
|
|
37
|
+
|
|
38
|
+
for char in text:
|
|
39
|
+
if char not in node.children:
|
|
40
|
+
break
|
|
41
|
+
node = node.children[char]
|
|
42
|
+
current_len += 1
|
|
43
|
+
if node.is_end:
|
|
44
|
+
last_match_value = node.value
|
|
45
|
+
last_match_len = current_len
|
|
46
|
+
|
|
47
|
+
return last_match_value, last_match_len
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
{
|
|
2
|
+
"aum": "ॐ",
|
|
3
|
+
"bibek": "विवेक",
|
|
4
|
+
"call": "कल",
|
|
5
|
+
"code": "कोड",
|
|
6
|
+
"cool": "कूल",
|
|
7
|
+
"facebook": "फेसबुक",
|
|
8
|
+
"fun": "फन",
|
|
9
|
+
"garage": "ग्यारेज",
|
|
10
|
+
"gas": "ग्यास",
|
|
11
|
+
"google": "गूगल",
|
|
12
|
+
"hola": "होला",
|
|
13
|
+
"hudai": "हुँदै",
|
|
14
|
+
"instagram": "इन्स्टाग्राम",
|
|
15
|
+
"kathmandu": "काठमाडौं",
|
|
16
|
+
"Kathmandu": "काठमाडौं",
|
|
17
|
+
"laptop": "ल्याप्टप",
|
|
18
|
+
"message": "मेसेज",
|
|
19
|
+
"messenger": "मेसेन्जर",
|
|
20
|
+
"nepal": "नेपाल",
|
|
21
|
+
"nepali": "नेपाली",
|
|
22
|
+
"number": "नम्बर",
|
|
23
|
+
"pandey": "पाण्डे",
|
|
24
|
+
"phone": "फोन",
|
|
25
|
+
"plan": "प्लान",
|
|
26
|
+
"radio": "रेडियो",
|
|
27
|
+
"sms": "एसएमएस",
|
|
28
|
+
"SMS": "एसएमएस",
|
|
29
|
+
"telecom": "टेलिकम",
|
|
30
|
+
"twitter": "ट्विटर",
|
|
31
|
+
"vivek": "विवेक",
|
|
32
|
+
"about": "अबाउट",
|
|
33
|
+
"address": "ठेगाना",
|
|
34
|
+
"bank": "बैंक",
|
|
35
|
+
"battery": "ब्याट्री",
|
|
36
|
+
"bus": "बस",
|
|
37
|
+
"camera": "क्यामेरा",
|
|
38
|
+
"chat": "च्याट",
|
|
39
|
+
"city": "सिटी",
|
|
40
|
+
"coffee": "कफी",
|
|
41
|
+
"computer": "कम्प्युटर",
|
|
42
|
+
"contact": "कन्ट्याक्ट",
|
|
43
|
+
"country": "देश",
|
|
44
|
+
"date": "मिति",
|
|
45
|
+
"email": "इमेल",
|
|
46
|
+
"file": "फाइल",
|
|
47
|
+
"food": "फुड",
|
|
48
|
+
"friend": "फ्रेन्ड",
|
|
49
|
+
"game": "गेम",
|
|
50
|
+
"hello": "हेलो",
|
|
51
|
+
"internet": "इन्टरनेट",
|
|
52
|
+
"location": "लोकेशन",
|
|
53
|
+
"market": "मार्केट",
|
|
54
|
+
"meeting": "मिटिङ",
|
|
55
|
+
"mobile": "मोबाइल",
|
|
56
|
+
"music": "संगीत",
|
|
57
|
+
"name": "नाम",
|
|
58
|
+
"photo": "फोटो",
|
|
59
|
+
"school": "स्कूल",
|
|
60
|
+
"shop": "पसल",
|
|
61
|
+
"station": "स्टेसन",
|
|
62
|
+
"taxi": "ट्याक्सी",
|
|
63
|
+
"time": "समय",
|
|
64
|
+
"video": "भिडियो",
|
|
65
|
+
"wifi": "वाईफाई",
|
|
66
|
+
"work": "काम"
|
|
67
|
+
}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nepali_unicoder
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: A Python package for Roman to Nepali (Devanagari) transliteration
|
|
5
|
+
Author-email: realsanjeev <075bei033.sanjeev@pcampus.edu.np>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/realsanjeev/nepali_unicoder
|
|
8
|
+
Keywords: nepali,unicoder,roman to devanagari,nepali transliteration,preeti to unicode
|
|
9
|
+
Classifier: Programming Language :: Python
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.7
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# Nepali Unicoder
|
|
20
|
+
|
|
21
|
+
A robust Python package for converting Romanized Nepali text and Preeti font text into Unicode Devanagari script. It uses a greedy matching algorithm for Roman transliteration and a two-phase conversion process for Preeti with contextual rules.
|
|
22
|
+
|
|
23
|
+
## Features
|
|
24
|
+
|
|
25
|
+
- **Accurate Transliteration**: Uses a greedy matching algorithm to prioritize longer phonetic matches (e.g., 'kha' is matched before 'k' and 'h').
|
|
26
|
+
- **Preeti Font Support**: Full support for Preeti to Unicode conversion with 30+ contextual rules for accurate transformation.
|
|
27
|
+
- **Smart Vowel Handling**: Distinguishes between independent vowels (e.g., 'aa' -> 'आ') and vowel signs/matras (e.g., 'ka' -> 'क', 'kaa' -> 'का').
|
|
28
|
+
- **Contextual Rules**: Handles complex Devanagari rules like reph positioning, matra reordering, and special character combinations.
|
|
29
|
+
- **Mixed Content Support**: Allows keeping English words or specific text in Roman script using `{}` blocks.
|
|
30
|
+
- **Customizable**: Supports custom word-level overrides via `word_maps.json`.
|
|
31
|
+
- **CLI Support**: Can be used directly from the command line.
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
You can install the package locally:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install nepali-unicoder
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Usage
|
|
42
|
+
|
|
43
|
+
### Command Line Interface (CLI)
|
|
44
|
+
|
|
45
|
+
You can use the converter directly from the terminal:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
# Direct argument
|
|
49
|
+
python -m nepali_unicoder "namaste"
|
|
50
|
+
# Output: नमस्ते
|
|
51
|
+
|
|
52
|
+
# Pipe input
|
|
53
|
+
echo "mero naam sanjeev ho" | python -m nepali_unicoder
|
|
54
|
+
# Output: मेरो नाम सन्जीव् हो
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### Python API
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from nepali_unicoder.convert import Converter
|
|
61
|
+
|
|
62
|
+
converter = Converter()
|
|
63
|
+
|
|
64
|
+
# Basic conversion
|
|
65
|
+
text = "namaste nepal"
|
|
66
|
+
print(converter.convert(text))
|
|
67
|
+
# Output: नमस्ते नेपाल
|
|
68
|
+
|
|
69
|
+
# Using 'as-is' blocks for English text
|
|
70
|
+
mixed_text = "mero naam {Sanjeev} ho"
|
|
71
|
+
print(converter.convert(mixed_text))
|
|
72
|
+
# Output: मेरो नाम Sanjeev हो
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Preeti Mode
|
|
76
|
+
|
|
77
|
+
Convert Preeti font text to Unicode with full support for contextual rules:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from nepali_unicoder.convert import Converter
|
|
81
|
+
|
|
82
|
+
# Create converter in Preeti mode
|
|
83
|
+
preeti_converter = Converter(mode="preeti")
|
|
84
|
+
|
|
85
|
+
# Basic conversion
|
|
86
|
+
preeti_text = "s{sf" # Preeti characters
|
|
87
|
+
print(preeti_converter.convert(preeti_text))
|
|
88
|
+
# Output: र्कर्का
|
|
89
|
+
|
|
90
|
+
# The converter handles:
|
|
91
|
+
# - Reph positioning: { → र् (moves before consonant)
|
|
92
|
+
# - Matra reordering: l (ि) moves after consonant
|
|
93
|
+
# - Special m transformations
|
|
94
|
+
# - Vowel combinations
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
#### Preeti Character Examples
|
|
98
|
+
|
|
99
|
+
| Preeti | Unicode | Description |
|
|
100
|
+
|--------|---------|-------------|
|
|
101
|
+
| `s` | `क` | Consonant ka |
|
|
102
|
+
| `s{` | `र्क` | Reph + ka (contextual) |
|
|
103
|
+
| `sl` | `कि` | ka + short i (reordered) |
|
|
104
|
+
| `qm` | `क्र` | Special m transformation |
|
|
105
|
+
| `!@#` | `१२३` | Nepali numbers |
|
|
106
|
+
|
|
107
|
+
#### CLI for Preeti
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
python -m nepali_unicoder --preeti "s{sf"
|
|
111
|
+
# Output: र्कर्का
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
## Transliteration Rules
|
|
116
|
+
|
|
117
|
+
- **Consonants**: `k` -> `क्`, `ka` -> `क`, `kh` -> `ख्`, `kha` -> `ख`
|
|
118
|
+
- **Vowels**: `a` -> `अ`, `aa` -> `आ`, `i` -> `इ`, `u` -> `उ`
|
|
119
|
+
- **Matras**: `ki` -> `कि`, `ko` -> `को`
|
|
120
|
+
- **Special**: `.` -> `।`, `..` -> `॥`
|
|
121
|
+
- **Numbers**: `0-9` -> `०-९` (Decimal points are preserved: `1.5` -> `१.५`)
|
|
122
|
+
|
|
123
|
+
## Advanced Usage
|
|
124
|
+
|
|
125
|
+
### Handling Complex Text
|
|
126
|
+
|
|
127
|
+
The converter handles mixed content gracefully. You can use `{}` to keep text as-is (e.g., for English words or code snippets).
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
text = "mero naam {Sanjeev} ho ra ma 12.5 barsa ko bhaye."
|
|
131
|
+
print(converter.convert(text))
|
|
132
|
+
# Output: मेरो नाम Sanjeev हो र म १२.५ बर्स को भए।
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
### Configuration
|
|
136
|
+
|
|
137
|
+
The package uses `word_maps.json` for custom word-level overrides, located in the `src/nepali_unicoder` directory.
|
|
138
|
+
|
|
139
|
+
1. **`word_maps.json`**: Defines custom word-level overrides. Use this for words that don't follow standard phonetic rules.
|
|
140
|
+
|
|
141
|
+
Example `word_maps.json`:
|
|
142
|
+
```json
|
|
143
|
+
{
|
|
144
|
+
"nepal": "नेपाल",
|
|
145
|
+
"kathamandu": "काठमाडौँ"
|
|
146
|
+
}
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
## Contribution
|
|
150
|
+
|
|
151
|
+
We welcome contributions! Here's how you can help:
|
|
152
|
+
|
|
153
|
+
1. **Clone the repository**:
|
|
154
|
+
```bash
|
|
155
|
+
git clone https://github.com/realsanjeev/nepali_unicoder.git
|
|
156
|
+
cd nepali_unicoder
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
2. **Set up a virtual environment**:
|
|
160
|
+
```bash
|
|
161
|
+
python3 -m venv .venv
|
|
162
|
+
source .venv/bin/activate
|
|
163
|
+
pip install -e .
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
3. **Run tests**:
|
|
167
|
+
```bash
|
|
168
|
+
python -m unittest discover tests
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
4. **Submit a Pull Request**: Create a new branch, make your changes, and submit a PR.
|
|
172
|
+
|
|
173
|
+
## Development
|
|
174
|
+
|
|
175
|
+
To run tests:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
python -m unittest discover tests
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
[MIT](./LICENSE)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
nepali_unicoder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
nepali_unicoder/__main__.py,sha256=m0-gSQW2HXfnCVWVLaqnW28KR1MA3y7U_iBCkc4Ur0I,790
|
|
3
|
+
nepali_unicoder/convert.py,sha256=st0O0zc_-UZiGjcHCo8RJUNkjjam1cmecs39VT4zlZ0,306
|
|
4
|
+
nepali_unicoder/engine.py,sha256=QVnYUnjRwlwkCg9PORaKoukMqjoOL30Pc5uiCp8C4TU,3267
|
|
5
|
+
nepali_unicoder/loader.py,sha256=EJnFpV1CL3yal5oqN_ojfPFAajWmntLiKdZS11lcirU,2602
|
|
6
|
+
nepali_unicoder/rules.py,sha256=0GjEEnNRtGWQEvrFhmjqnYW953taAHidSdNLcFA0Q9o,7051
|
|
7
|
+
nepali_unicoder/tokenizer.py,sha256=nGm-1hO_9WeQNqyxEj5pmIcLYrw1uBuND1jIJ_wRzKk,2462
|
|
8
|
+
nepali_unicoder/trie.py,sha256=gpOnDs3AFqf8Q4DjSYCcUnu6ep6yFjv6KyWAA_k8cew,1355
|
|
9
|
+
nepali_unicoder/word_maps.json,sha256=z6di4YZn_EGvkKvNa9bffLcYw-esTAKYA2PJPIqBqrk,2158
|
|
10
|
+
nepali_unicoder-0.1.1.dist-info/licenses/LICENSE,sha256=nWolli2fvrHF2ActODUDH0QjSKZf8EDb-M-vp74xa9I,1068
|
|
11
|
+
nepali_unicoder-0.1.1.dist-info/METADATA,sha256=kEYf-Jp4bjO1JER2o-GaBoEPas7loOXJj9T7vqdmpYY,5319
|
|
12
|
+
nepali_unicoder-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
13
|
+
nepali_unicoder-0.1.1.dist-info/top_level.txt,sha256=hoBnas3ztt0HTE0NM5jWgBsqDetj9OPg4rMwMPTqM3I,16
|
|
14
|
+
nepali_unicoder-0.1.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 RealSanjeev
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
nepali_unicoder
|