nepali_unicoder 0.1.2__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nepali_unicoder-0.2.1/MANIFEST.in +2 -0
- {nepali_unicoder-0.1.2/src/nepali_unicoder.egg-info → nepali_unicoder-0.2.1}/PKG-INFO +14 -10
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/README.md +11 -7
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/pyproject.toml +6 -3
- nepali_unicoder-0.2.1/src/nepali_unicoder/__init__.py +3 -0
- nepali_unicoder-0.2.1/src/nepali_unicoder/__main__.py +41 -0
- nepali_unicoder-0.2.1/src/nepali_unicoder/data/preeti_rules.json +230 -0
- nepali_unicoder-0.2.1/src/nepali_unicoder/data/roman_rules.json +112 -0
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/src/nepali_unicoder/engine.py +14 -5
- nepali_unicoder-0.2.1/src/nepali_unicoder/loader.py +120 -0
- nepali_unicoder-0.2.1/src/nepali_unicoder/rules.py +2 -0
- nepali_unicoder-0.2.1/src/nepali_unicoder/tokenizer.py +93 -0
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/src/nepali_unicoder/trie.py +8 -2
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1/src/nepali_unicoder.egg-info}/PKG-INFO +14 -10
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/src/nepali_unicoder.egg-info/SOURCES.txt +4 -1
- nepali_unicoder-0.2.1/src/nepali_unicoder.egg-info/entry_points.txt +2 -0
- nepali_unicoder-0.2.1/tests/test_components.py +141 -0
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/tests/test_conversion.py +59 -2
- nepali_unicoder-0.1.2/MANIFEST.in +0 -1
- nepali_unicoder-0.1.2/src/nepali_unicoder/__init__.py +0 -0
- nepali_unicoder-0.1.2/src/nepali_unicoder/__main__.py +0 -32
- nepali_unicoder-0.1.2/src/nepali_unicoder/loader.py +0 -84
- nepali_unicoder-0.1.2/src/nepali_unicoder/rules.py +0 -279
- nepali_unicoder-0.1.2/src/nepali_unicoder/tokenizer.py +0 -74
- nepali_unicoder-0.1.2/tests/test_components.py +0 -70
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/LICENSE +0 -0
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/setup.cfg +0 -0
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/src/nepali_unicoder/convert.py +0 -0
- {nepali_unicoder-0.1.2/src/nepali_unicoder → nepali_unicoder-0.2.1/src/nepali_unicoder/data}/word_maps.json +0 -0
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/src/nepali_unicoder.egg-info/dependency_links.txt +0 -0
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/src/nepali_unicoder.egg-info/requires.txt +0 -0
- {nepali_unicoder-0.1.2 → nepali_unicoder-0.2.1}/src/nepali_unicoder.egg-info/top_level.txt +0 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
|
-
Name:
|
|
3
|
-
Version: 0.1
|
|
2
|
+
Name: nepali-unicoder
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: A Python package for Roman to Nepali (Devanagari) transliteration
|
|
5
5
|
Author-email: realsanjeev <075bei033.sanjeev@pcampus.edu.np>
|
|
6
|
-
License
|
|
6
|
+
License: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/realsanjeev/nepali_unicoder
|
|
8
8
|
Project-URL: Documentation, https://realsanjeev.github.io/nepali_unicoder/
|
|
9
9
|
Project-URL: Bug Tracker, https://github.com/realsanjeev/nepali_unicoder/issues
|
|
@@ -54,13 +54,17 @@ pip install nepali-unicoder
|
|
|
54
54
|
You can use the converter directly from the terminal:
|
|
55
55
|
|
|
56
56
|
```bash
|
|
57
|
-
#
|
|
57
|
+
# Using the installed command
|
|
58
|
+
nepali-unicoder "namaste"
|
|
59
|
+
# Output: नमस्ते
|
|
60
|
+
|
|
61
|
+
# Or using python module
|
|
58
62
|
python -m nepali_unicoder "namaste"
|
|
59
63
|
# Output: नमस्ते
|
|
60
64
|
|
|
61
65
|
# Pipe input
|
|
62
|
-
echo "mero naam sanjeev ho" |
|
|
63
|
-
# Output: मेरो
|
|
66
|
+
echo "mero naam sanjeev ho" | nepali-unicoder
|
|
67
|
+
# Output: मेरो नाम् सन्जीव् हो
|
|
64
68
|
```
|
|
65
69
|
|
|
66
70
|
### Python API
|
|
@@ -78,7 +82,7 @@ print(converter.convert(text))
|
|
|
78
82
|
# Using 'as-is' blocks for English text
|
|
79
83
|
mixed_text = "mero naam {Sanjeev} ho"
|
|
80
84
|
print(converter.convert(mixed_text))
|
|
81
|
-
# Output: मेरो
|
|
85
|
+
# Output: मेरो नाम् Sanjeev हो
|
|
82
86
|
```
|
|
83
87
|
|
|
84
88
|
### Preeti Mode
|
|
@@ -94,7 +98,7 @@ preeti_converter = Converter(mode="preeti")
|
|
|
94
98
|
# Basic conversion
|
|
95
99
|
preeti_text = "s{sf" # Preeti characters
|
|
96
100
|
print(preeti_converter.convert(preeti_text))
|
|
97
|
-
# Output:
|
|
101
|
+
# Output: र्कका
|
|
98
102
|
|
|
99
103
|
# The converter handles:
|
|
100
104
|
# - Reph positioning: { → र् (moves before consonant)
|
|
@@ -131,7 +135,7 @@ print(preeti_converter.convert(preeti_text))
|
|
|
131
135
|
|
|
132
136
|
```bash
|
|
133
137
|
python -m nepali_unicoder --preeti "s{sf"
|
|
134
|
-
# Output:
|
|
138
|
+
# Output: र्कका
|
|
135
139
|
```
|
|
136
140
|
|
|
137
141
|
|
|
@@ -152,7 +156,7 @@ The converter handles mixed content gracefully. You can use `{}` to keep text as
|
|
|
152
156
|
```python
|
|
153
157
|
text = "mero naam {Sanjeev} ho ra ma 12.5 barsa ko bhaye."
|
|
154
158
|
print(converter.convert(text))
|
|
155
|
-
# Output: मेरो
|
|
159
|
+
# Output: मेरो नाम् Sanjeev हो र म १२.५ बर्स को भये।
|
|
156
160
|
```
|
|
157
161
|
|
|
158
162
|
### Configuration
|
|
@@ -32,13 +32,17 @@ pip install nepali-unicoder
|
|
|
32
32
|
You can use the converter directly from the terminal:
|
|
33
33
|
|
|
34
34
|
```bash
|
|
35
|
-
#
|
|
35
|
+
# Using the installed command
|
|
36
|
+
nepali-unicoder "namaste"
|
|
37
|
+
# Output: नमस्ते
|
|
38
|
+
|
|
39
|
+
# Or using python module
|
|
36
40
|
python -m nepali_unicoder "namaste"
|
|
37
41
|
# Output: नमस्ते
|
|
38
42
|
|
|
39
43
|
# Pipe input
|
|
40
|
-
echo "mero naam sanjeev ho" |
|
|
41
|
-
# Output: मेरो
|
|
44
|
+
echo "mero naam sanjeev ho" | nepali-unicoder
|
|
45
|
+
# Output: मेरो नाम् सन्जीव् हो
|
|
42
46
|
```
|
|
43
47
|
|
|
44
48
|
### Python API
|
|
@@ -56,7 +60,7 @@ print(converter.convert(text))
|
|
|
56
60
|
# Using 'as-is' blocks for English text
|
|
57
61
|
mixed_text = "mero naam {Sanjeev} ho"
|
|
58
62
|
print(converter.convert(mixed_text))
|
|
59
|
-
# Output: मेरो
|
|
63
|
+
# Output: मेरो नाम् Sanjeev हो
|
|
60
64
|
```
|
|
61
65
|
|
|
62
66
|
### Preeti Mode
|
|
@@ -72,7 +76,7 @@ preeti_converter = Converter(mode="preeti")
|
|
|
72
76
|
# Basic conversion
|
|
73
77
|
preeti_text = "s{sf" # Preeti characters
|
|
74
78
|
print(preeti_converter.convert(preeti_text))
|
|
75
|
-
# Output:
|
|
79
|
+
# Output: र्कका
|
|
76
80
|
|
|
77
81
|
# The converter handles:
|
|
78
82
|
# - Reph positioning: { → र् (moves before consonant)
|
|
@@ -109,7 +113,7 @@ print(preeti_converter.convert(preeti_text))
|
|
|
109
113
|
|
|
110
114
|
```bash
|
|
111
115
|
python -m nepali_unicoder --preeti "s{sf"
|
|
112
|
-
# Output:
|
|
116
|
+
# Output: र्कका
|
|
113
117
|
```
|
|
114
118
|
|
|
115
119
|
|
|
@@ -130,7 +134,7 @@ The converter handles mixed content gracefully. You can use `{}` to keep text as
|
|
|
130
134
|
```python
|
|
131
135
|
text = "mero naam {Sanjeev} ho ra ma 12.5 barsa ko bhaye."
|
|
132
136
|
print(converter.convert(text))
|
|
133
|
-
# Output: मेरो
|
|
137
|
+
# Output: मेरो नाम् Sanjeev हो र म १२.५ बर्स को भये।
|
|
134
138
|
```
|
|
135
139
|
|
|
136
140
|
### Configuration
|
|
@@ -3,14 +3,14 @@ requires = ["setuptools>=61.0"]
|
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
|
-
name = "
|
|
7
|
-
version = "0.1
|
|
6
|
+
name = "nepali-unicoder"
|
|
7
|
+
version = "0.2.1"
|
|
8
8
|
description = "A Python package for Roman to Nepali (Devanagari) transliteration"
|
|
9
9
|
authors = [
|
|
10
10
|
{ name="realsanjeev", email="075bei033.sanjeev@pcampus.edu.np" },
|
|
11
11
|
]
|
|
12
12
|
readme = "README.md"
|
|
13
|
-
license = "MIT"
|
|
13
|
+
license = { text = "MIT" }
|
|
14
14
|
requires-python = ">=3.7"
|
|
15
15
|
dependencies = []
|
|
16
16
|
classifiers = [
|
|
@@ -20,6 +20,9 @@ classifiers = [
|
|
|
20
20
|
]
|
|
21
21
|
keywords = ["nepali", "unicoder", "roman to devanagari", "nepali transliteration", "preeti to unicode"]
|
|
22
22
|
|
|
23
|
+
[project.scripts]
|
|
24
|
+
nepali-unicoder = "nepali_unicoder.__main__:main"
|
|
25
|
+
|
|
23
26
|
|
|
24
27
|
[project.urls]
|
|
25
28
|
"Homepage" = "https://github.com/realsanjeev/nepali_unicoder"
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
from nepali_unicoder.convert import Converter
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def main():
|
|
8
|
+
parser = argparse.ArgumentParser(
|
|
9
|
+
prog="python -m nepali_unicoder",
|
|
10
|
+
description="Convert Romanized Nepali or Preeti font text to Unicode Devanagari.",
|
|
11
|
+
)
|
|
12
|
+
parser.add_argument(
|
|
13
|
+
"text",
|
|
14
|
+
nargs="*",
|
|
15
|
+
help="The text to convert. If omitted, reads from stdin.",
|
|
16
|
+
)
|
|
17
|
+
parser.add_argument(
|
|
18
|
+
"--preeti",
|
|
19
|
+
action="store_true",
|
|
20
|
+
help="Enable Preeti to Unicode conversion mode.",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
args = parser.parse_args()
|
|
24
|
+
|
|
25
|
+
# Determine input text
|
|
26
|
+
if args.text:
|
|
27
|
+
input_text = " ".join(args.text)
|
|
28
|
+
elif not sys.stdin.isatty():
|
|
29
|
+
input_text = sys.stdin.read().strip()
|
|
30
|
+
else:
|
|
31
|
+
parser.print_help()
|
|
32
|
+
return
|
|
33
|
+
|
|
34
|
+
mode = "preeti" if args.preeti else "roman"
|
|
35
|
+
converter = Converter(mode=mode)
|
|
36
|
+
result = converter.convert(input_text)
|
|
37
|
+
print(result)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
if __name__ == "__main__":
|
|
41
|
+
main()
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
{
|
|
2
|
+
"mappings": {
|
|
3
|
+
"a": "ब",
|
|
4
|
+
"b": "द",
|
|
5
|
+
"c": "अ",
|
|
6
|
+
"d": "म",
|
|
7
|
+
"e": "भ",
|
|
8
|
+
"f": "ा",
|
|
9
|
+
"g": "न",
|
|
10
|
+
"h": "ज",
|
|
11
|
+
"i": "ष्",
|
|
12
|
+
"j": "व",
|
|
13
|
+
"k": "प",
|
|
14
|
+
"l": "ि",
|
|
15
|
+
"m": "m",
|
|
16
|
+
"n": "ल",
|
|
17
|
+
"o": "य",
|
|
18
|
+
"p": "उ",
|
|
19
|
+
"q": "त्र",
|
|
20
|
+
"r": "च",
|
|
21
|
+
"s": "क",
|
|
22
|
+
"t": "त",
|
|
23
|
+
"u": "ग",
|
|
24
|
+
"v": "ख",
|
|
25
|
+
"w": "ध",
|
|
26
|
+
"x": "ह",
|
|
27
|
+
"y": "थ",
|
|
28
|
+
"z": "श",
|
|
29
|
+
"A": "ब्",
|
|
30
|
+
"B": "द्य",
|
|
31
|
+
"C": "ऋ",
|
|
32
|
+
"D": "म्",
|
|
33
|
+
"E": "भ्",
|
|
34
|
+
"F": "ँ",
|
|
35
|
+
"G": "न्",
|
|
36
|
+
"H": "ज्",
|
|
37
|
+
"I": "क्ष्",
|
|
38
|
+
"J": "व्",
|
|
39
|
+
"K": "प्",
|
|
40
|
+
"L": "ी",
|
|
41
|
+
"M": "ः",
|
|
42
|
+
"N": "ल्",
|
|
43
|
+
"O": "इ",
|
|
44
|
+
"P": "ए",
|
|
45
|
+
"Q": "त्त",
|
|
46
|
+
"R": "च्",
|
|
47
|
+
"S": "क्",
|
|
48
|
+
"T": "त्",
|
|
49
|
+
"U": "ग्",
|
|
50
|
+
"V": "ख्",
|
|
51
|
+
"W": "ध्",
|
|
52
|
+
"X": "ह्",
|
|
53
|
+
"Y": "थ्",
|
|
54
|
+
"Z": "श्",
|
|
55
|
+
"0": "ण्",
|
|
56
|
+
"1": "ज्ञ",
|
|
57
|
+
"2": "द्द",
|
|
58
|
+
"3": "घ",
|
|
59
|
+
"4": "द्ध",
|
|
60
|
+
"5": "छ",
|
|
61
|
+
"6": "ट",
|
|
62
|
+
"7": "ठ",
|
|
63
|
+
"8": "ड",
|
|
64
|
+
"9": "ढ",
|
|
65
|
+
"!": "१",
|
|
66
|
+
"@": "२",
|
|
67
|
+
"#": "३",
|
|
68
|
+
"$": "४",
|
|
69
|
+
"%": "५",
|
|
70
|
+
"^": "६",
|
|
71
|
+
"&": "७",
|
|
72
|
+
"*": "८",
|
|
73
|
+
"(": "९",
|
|
74
|
+
")": "०",
|
|
75
|
+
"[": "ृ",
|
|
76
|
+
"]": "े",
|
|
77
|
+
"{": "{",
|
|
78
|
+
"}": "ै",
|
|
79
|
+
"|": "्र",
|
|
80
|
+
";": "स",
|
|
81
|
+
":": "स्",
|
|
82
|
+
"'": "ु",
|
|
83
|
+
"\"": "ू",
|
|
84
|
+
",": ",",
|
|
85
|
+
".": "।",
|
|
86
|
+
"/": "र",
|
|
87
|
+
"<": "?",
|
|
88
|
+
">": "श्र",
|
|
89
|
+
"?": "रु",
|
|
90
|
+
"\\": "्",
|
|
91
|
+
"`": "ञ",
|
|
92
|
+
"~": "ञ्",
|
|
93
|
+
"_": ")",
|
|
94
|
+
"-": "(",
|
|
95
|
+
"=": ".",
|
|
96
|
+
"...": "...",
|
|
97
|
+
"+": "ं",
|
|
98
|
+
"Ù": ";",
|
|
99
|
+
"Ú": ":",
|
|
100
|
+
"«": "्र",
|
|
101
|
+
"»": "्र",
|
|
102
|
+
"¿": "रू",
|
|
103
|
+
"å": "द्व",
|
|
104
|
+
"ˆ": "फ्",
|
|
105
|
+
"ª": "ङ",
|
|
106
|
+
"æ": "“",
|
|
107
|
+
"Æ": "”",
|
|
108
|
+
"¥": "र्",
|
|
109
|
+
"¶": "ठ्ठ",
|
|
110
|
+
"§": "ट्ट",
|
|
111
|
+
"£": "घ्",
|
|
112
|
+
"Ë": "ङ्ग",
|
|
113
|
+
"Í": "ङ्क",
|
|
114
|
+
"‰": "झ्"
|
|
115
|
+
},
|
|
116
|
+
"post_rules": [
|
|
117
|
+
[
|
|
118
|
+
"्ा",
|
|
119
|
+
""
|
|
120
|
+
],
|
|
121
|
+
[
|
|
122
|
+
"(त्र|त्त)([^उभप]+?)m",
|
|
123
|
+
"\\1m\\2"
|
|
124
|
+
],
|
|
125
|
+
[
|
|
126
|
+
"त्रm",
|
|
127
|
+
"क्र"
|
|
128
|
+
],
|
|
129
|
+
[
|
|
130
|
+
"त्तm",
|
|
131
|
+
"क्त"
|
|
132
|
+
],
|
|
133
|
+
[
|
|
134
|
+
"([^उभप]+?)m",
|
|
135
|
+
"m\\1"
|
|
136
|
+
],
|
|
137
|
+
[
|
|
138
|
+
"उm",
|
|
139
|
+
"ऊ"
|
|
140
|
+
],
|
|
141
|
+
[
|
|
142
|
+
"भm",
|
|
143
|
+
"झ"
|
|
144
|
+
],
|
|
145
|
+
[
|
|
146
|
+
"पm",
|
|
147
|
+
"फ"
|
|
148
|
+
],
|
|
149
|
+
[
|
|
150
|
+
"इ{",
|
|
151
|
+
"ई"
|
|
152
|
+
],
|
|
153
|
+
[
|
|
154
|
+
"ि([ाीुूृेैोौंःँ]*((.्)*[^्]))",
|
|
155
|
+
"\\1ि"
|
|
156
|
+
],
|
|
157
|
+
[
|
|
158
|
+
"(.[ािीुूृेैोौंःँ]*?){",
|
|
159
|
+
"{\\1"
|
|
160
|
+
],
|
|
161
|
+
[
|
|
162
|
+
"((.्)*){",
|
|
163
|
+
"{\\1"
|
|
164
|
+
],
|
|
165
|
+
[
|
|
166
|
+
"{",
|
|
167
|
+
"र्"
|
|
168
|
+
],
|
|
169
|
+
[
|
|
170
|
+
"([ाीुूृेैोौंःँ]+?)(्(.्)*[^्])",
|
|
171
|
+
"\\2\\1"
|
|
172
|
+
],
|
|
173
|
+
[
|
|
174
|
+
"्([ाीुूृेैोौंःँ]+?)((.्)*[^्])",
|
|
175
|
+
"्\\2\\1"
|
|
176
|
+
],
|
|
177
|
+
[
|
|
178
|
+
"([ंँ])([ािीुूृेैोौः]*)",
|
|
179
|
+
"\\2\\1"
|
|
180
|
+
],
|
|
181
|
+
[
|
|
182
|
+
"([ँंेैुू])\\1",
|
|
183
|
+
"\\1"
|
|
184
|
+
],
|
|
185
|
+
[
|
|
186
|
+
"^ः",
|
|
187
|
+
":"
|
|
188
|
+
],
|
|
189
|
+
[
|
|
190
|
+
" ः",
|
|
191
|
+
":"
|
|
192
|
+
],
|
|
193
|
+
[
|
|
194
|
+
"टृ",
|
|
195
|
+
"ट्ट"
|
|
196
|
+
],
|
|
197
|
+
[
|
|
198
|
+
"ेा",
|
|
199
|
+
"ाे"
|
|
200
|
+
],
|
|
201
|
+
[
|
|
202
|
+
"ैा",
|
|
203
|
+
"ाै"
|
|
204
|
+
],
|
|
205
|
+
[
|
|
206
|
+
"अाे",
|
|
207
|
+
"ओ"
|
|
208
|
+
],
|
|
209
|
+
[
|
|
210
|
+
"अाै",
|
|
211
|
+
"औ"
|
|
212
|
+
],
|
|
213
|
+
[
|
|
214
|
+
"अा",
|
|
215
|
+
"आ"
|
|
216
|
+
],
|
|
217
|
+
[
|
|
218
|
+
"एे",
|
|
219
|
+
"ऐ"
|
|
220
|
+
],
|
|
221
|
+
[
|
|
222
|
+
"ाे",
|
|
223
|
+
"ो"
|
|
224
|
+
],
|
|
225
|
+
[
|
|
226
|
+
"ाै",
|
|
227
|
+
"ौ"
|
|
228
|
+
]
|
|
229
|
+
]
|
|
230
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
{
|
|
2
|
+
"consonants": {
|
|
3
|
+
"k": "क",
|
|
4
|
+
"kh": "ख",
|
|
5
|
+
"g": "ग",
|
|
6
|
+
"gh": "घ",
|
|
7
|
+
"ng": "ङ",
|
|
8
|
+
"ch": "च",
|
|
9
|
+
"chh": "छ",
|
|
10
|
+
"j": "ज",
|
|
11
|
+
"jh": "झ",
|
|
12
|
+
"ny": "ञ",
|
|
13
|
+
"T": "ट",
|
|
14
|
+
"Th": "ठ",
|
|
15
|
+
"D": "ड",
|
|
16
|
+
"Dh": "ढ",
|
|
17
|
+
"N": "ण",
|
|
18
|
+
"t": "त",
|
|
19
|
+
"th": "थ",
|
|
20
|
+
"d": "द",
|
|
21
|
+
"dh": "ध",
|
|
22
|
+
"n": "न",
|
|
23
|
+
"p": "प",
|
|
24
|
+
"ph": "फ",
|
|
25
|
+
"f": "फ",
|
|
26
|
+
"b": "ब",
|
|
27
|
+
"bh": "भ",
|
|
28
|
+
"m": "म",
|
|
29
|
+
"y": "य",
|
|
30
|
+
"r": "र",
|
|
31
|
+
"l": "ल",
|
|
32
|
+
"v": "व",
|
|
33
|
+
"w": "व",
|
|
34
|
+
"sh": "श",
|
|
35
|
+
"Sh": "ष",
|
|
36
|
+
"s": "स",
|
|
37
|
+
"h": "ह",
|
|
38
|
+
"ksh": "क्ष",
|
|
39
|
+
"gy": "ज्ञ",
|
|
40
|
+
"shr": "श्र",
|
|
41
|
+
"tr": "त्र"
|
|
42
|
+
|
|
43
|
+
},
|
|
44
|
+
"vowels": {
|
|
45
|
+
"a": "अ",
|
|
46
|
+
"aa": "आ",
|
|
47
|
+
"A": "आ",
|
|
48
|
+
"i": "इ",
|
|
49
|
+
"I": "इ",
|
|
50
|
+
"ee": "ई",
|
|
51
|
+
"ii": "ई",
|
|
52
|
+
"u": "उ",
|
|
53
|
+
"U": "उ",
|
|
54
|
+
"oo": "ऊ",
|
|
55
|
+
"uu": "ऊ",
|
|
56
|
+
"Ri": "ऋ",
|
|
57
|
+
"R": "ऋ",
|
|
58
|
+
"Ree": "ॠ",
|
|
59
|
+
"lRi": "ऌ",
|
|
60
|
+
"e": "ए",
|
|
61
|
+
"ai": "ऐ",
|
|
62
|
+
"o": "ओ",
|
|
63
|
+
"au": "औ"
|
|
64
|
+
},
|
|
65
|
+
"matras": {
|
|
66
|
+
"a": "",
|
|
67
|
+
"aa": "ा",
|
|
68
|
+
"A": "ा",
|
|
69
|
+
"i": "ि",
|
|
70
|
+
"I": "ि",
|
|
71
|
+
"ee": "ी",
|
|
72
|
+
"ii": "ी",
|
|
73
|
+
"u": "ु",
|
|
74
|
+
"U": "ु",
|
|
75
|
+
"oo": "ू",
|
|
76
|
+
"uu": "ू",
|
|
77
|
+
"Ri": "ृ",
|
|
78
|
+
"R": "ृ",
|
|
79
|
+
"Ree": "ॄ",
|
|
80
|
+
"e": "े",
|
|
81
|
+
"ai": "ै",
|
|
82
|
+
"o": "ो",
|
|
83
|
+
"au": "ौ"
|
|
84
|
+
},
|
|
85
|
+
"special": {
|
|
86
|
+
"om": "ॐ",
|
|
87
|
+
"Om": "ॐ",
|
|
88
|
+
"aum": "ॐ",
|
|
89
|
+
".": "।",
|
|
90
|
+
"..": "॥",
|
|
91
|
+
"|": "।",
|
|
92
|
+
"||": "॥",
|
|
93
|
+
"M": "ं",
|
|
94
|
+
"H": "ः",
|
|
95
|
+
"~": "ँ"
|
|
96
|
+
},
|
|
97
|
+
"compound_consonants": {
|
|
98
|
+
|
|
99
|
+
},
|
|
100
|
+
"digits": {
|
|
101
|
+
"0": "०",
|
|
102
|
+
"1": "१",
|
|
103
|
+
"2": "२",
|
|
104
|
+
"3": "३",
|
|
105
|
+
"4": "४",
|
|
106
|
+
"5": "५",
|
|
107
|
+
"6": "६",
|
|
108
|
+
"7": "७",
|
|
109
|
+
"8": "८",
|
|
110
|
+
"9": "९"
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -2,18 +2,23 @@ import re
|
|
|
2
2
|
from typing import Optional
|
|
3
3
|
|
|
4
4
|
from nepali_unicoder.loader import PreetiLoader, RuleLoader
|
|
5
|
-
from nepali_unicoder.rules import PREETI_TO_UNICODE_MAPPING
|
|
6
5
|
from nepali_unicoder.tokenizer import Tokenizer
|
|
7
6
|
from nepali_unicoder.trie import Trie
|
|
8
7
|
|
|
9
8
|
|
|
10
9
|
class Engine:
|
|
10
|
+
VALID_MODES = {"roman", "preeti"}
|
|
11
|
+
|
|
11
12
|
def __init__(
|
|
12
13
|
self,
|
|
13
14
|
trie: Optional[Trie] = None,
|
|
14
15
|
tokenizer: Optional[Tokenizer] = None,
|
|
15
16
|
mode: str = "roman",
|
|
16
17
|
):
|
|
18
|
+
if mode not in self.VALID_MODES:
|
|
19
|
+
raise ValueError(
|
|
20
|
+
f"Invalid mode {mode!r}. Expected one of {sorted(self.VALID_MODES)}"
|
|
21
|
+
)
|
|
17
22
|
self.mode = mode
|
|
18
23
|
self.post_rules = []
|
|
19
24
|
|
|
@@ -21,7 +26,11 @@ class Engine:
|
|
|
21
26
|
if mode == "preeti":
|
|
22
27
|
loader = PreetiLoader()
|
|
23
28
|
# Load post-processing rules for Preeti mode
|
|
24
|
-
|
|
29
|
+
raw_rules = loader.get_post_rules()
|
|
30
|
+
self.post_rules = [
|
|
31
|
+
(re.compile(pattern), replacement)
|
|
32
|
+
for pattern, replacement in raw_rules
|
|
33
|
+
]
|
|
25
34
|
else:
|
|
26
35
|
loader = RuleLoader()
|
|
27
36
|
self.trie = loader.load()
|
|
@@ -69,8 +78,8 @@ class Engine:
|
|
|
69
78
|
|
|
70
79
|
while idx < chunk_len:
|
|
71
80
|
# Try to find longest match starting at idx
|
|
72
|
-
|
|
73
|
-
match_val, match_len = self.trie.longest_match(
|
|
81
|
+
# Pass chunk and idx directly to avoid slicing
|
|
82
|
+
match_val, match_len = self.trie.longest_match(chunk, idx)
|
|
74
83
|
|
|
75
84
|
if match_val:
|
|
76
85
|
result.append(match_val)
|
|
@@ -94,5 +103,5 @@ class Engine:
|
|
|
94
103
|
Used for Preeti mode to handle contextual transformations.
|
|
95
104
|
"""
|
|
96
105
|
for pattern, replacement in self.post_rules:
|
|
97
|
-
text =
|
|
106
|
+
text = pattern.sub(replacement, text)
|
|
98
107
|
return text
|