nepali_unicoder 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ from nepali_unicoder.convert import Converter
2
+
3
+ __all__ = ["Converter"]
@@ -0,0 +1,41 @@
1
+ import argparse
2
+ import sys
3
+
4
+ from nepali_unicoder.convert import Converter
5
+
6
+
7
+ def main():
8
+ parser = argparse.ArgumentParser(
9
+ prog="python -m nepali_unicoder",
10
+ description="Convert Romanized Nepali or Preeti font text to Unicode Devanagari.",
11
+ )
12
+ parser.add_argument(
13
+ "text",
14
+ nargs="*",
15
+ help="The text to convert. If omitted, reads from stdin.",
16
+ )
17
+ parser.add_argument(
18
+ "--preeti",
19
+ action="store_true",
20
+ help="Enable Preeti to Unicode conversion mode.",
21
+ )
22
+
23
+ args = parser.parse_args()
24
+
25
+ # Determine input text
26
+ if args.text:
27
+ input_text = " ".join(args.text)
28
+ elif not sys.stdin.isatty():
29
+ input_text = sys.stdin.read().strip()
30
+ else:
31
+ parser.print_help()
32
+ return
33
+
34
+ mode = "preeti" if args.preeti else "roman"
35
+ converter = Converter(mode=mode)
36
+ result = converter.convert(input_text)
37
+ print(result)
38
+
39
+
40
+ if __name__ == "__main__":
41
+ main()
@@ -0,0 +1,13 @@
1
+ from nepali_unicoder.engine import Engine
2
+
3
+
4
+ class Converter:
5
+ """
6
+ Wrapper around Engine for backward compatibility.
7
+ """
8
+
9
+ def __init__(self, mode: str = "roman"):
10
+ self.engine = Engine(mode=mode)
11
+
12
+ def convert(self, text: str) -> str:
13
+ return self.engine.transliterate(text)
@@ -0,0 +1,226 @@
1
+ {
2
+ "mappings": {
3
+ "a": "ब",
4
+ "b": "द",
5
+ "c": "अ",
6
+ "d": "म",
7
+ "e": "भ",
8
+ "f": "ा",
9
+ "g": "न",
10
+ "h": "ज",
11
+ "i": "ष्",
12
+ "j": "व",
13
+ "k": "प",
14
+ "l": "ि",
15
+ "m": "m",
16
+ "n": "ल",
17
+ "o": "य",
18
+ "p": "उ",
19
+ "q": "त्र",
20
+ "r": "च",
21
+ "s": "क",
22
+ "t": "त",
23
+ "u": "ग",
24
+ "v": "ख",
25
+ "w": "ध",
26
+ "x": "ह",
27
+ "y": "थ",
28
+ "z": "श",
29
+ "A": "ब्",
30
+ "B": "द्य",
31
+ "C": "ऋ",
32
+ "D": "म्",
33
+ "E": "भ्",
34
+ "F": "ँ",
35
+ "G": "न्",
36
+ "H": "ज्",
37
+ "I": "क्ष्",
38
+ "J": "व्",
39
+ "K": "प्",
40
+ "L": "ी",
41
+ "M": "ः",
42
+ "N": "ल्",
43
+ "O": "इ",
44
+ "P": "ए",
45
+ "Q": "त्त",
46
+ "R": "च्",
47
+ "S": "क्",
48
+ "T": "त्",
49
+ "U": "ग्",
50
+ "V": "ख्",
51
+ "W": "ध्",
52
+ "X": "ह्",
53
+ "Y": "थ्",
54
+ "Z": "श्",
55
+ "0": "ण्",
56
+ "1": "ज्ञ",
57
+ "2": "द्द",
58
+ "3": "घ",
59
+ "4": "द्ध",
60
+ "5": "छ",
61
+ "6": "ट",
62
+ "7": "ठ",
63
+ "8": "ड",
64
+ "9": "ढ",
65
+ "!": "१",
66
+ "@": "२",
67
+ "#": "३",
68
+ "$": "४",
69
+ "%": "५",
70
+ "^": "६",
71
+ "&": "७",
72
+ "*": "८",
73
+ "(": "९",
74
+ ")": "०",
75
+ "[": "ृ",
76
+ "]": "े",
77
+ "{": "{",
78
+ "}": "ै",
79
+ "|": "्र",
80
+ ";": "स",
81
+ ":": "स्",
82
+ "'": "ु",
83
+ "\"": "ू",
84
+ ",": ",",
85
+ ".": "।",
86
+ "/": "र",
87
+ "<": "?",
88
+ ">": "श्र",
89
+ "?": "रु",
90
+ "\\": "्",
91
+ "`": "ञ",
92
+ "~": "ञ्",
93
+ "_": ")",
94
+ "-": "(",
95
+ "=": ".",
96
+ "...": "...",
97
+ "+": "ं",
98
+ "Ù": ";",
99
+ "Ú": ":",
100
+ "«": "्र",
101
+ "»": "्र",
102
+ "¿": "रू",
103
+ "å": "द्व",
104
+ "ˆ": "फ्",
105
+ "ª": "ङ",
106
+ "æ": "“",
107
+ "Æ": "”",
108
+ "¥": "र्",
109
+ "¶": "ठ्ठ",
110
+ "§": "ट्ट",
111
+ "£": "घ्",
112
+ "Ë": "ङ्ग",
113
+ "Í": "ङ्क",
114
+ "‰": "झ्"
115
+ },
116
+ "post_rules": [
117
+ [
118
+ "्ा",
119
+ ""
120
+ ],
121
+ [
122
+ "(त्र|त्त)([^उभप]+?)m",
123
+ "\\1m\\2"
124
+ ],
125
+ [
126
+ "त्रm",
127
+ "क्र"
128
+ ],
129
+ [
130
+ "त्तm",
131
+ "क्त"
132
+ ],
133
+ [
134
+ "([^उभप]+?)m",
135
+ "m\\1"
136
+ ],
137
+ [
138
+ "उm",
139
+ "ऊ"
140
+ ],
141
+ [
142
+ "भm",
143
+ "झ"
144
+ ],
145
+ [
146
+ "पm",
147
+ "फ"
148
+ ],
149
+ [
150
+ "इ{",
151
+ "ई"
152
+ ],
153
+ [
154
+ "ि((.्)*[^्])",
155
+ "\\1ि"
156
+ ],
157
+ [
158
+ "(.[ािीुूृेैोौंःँ]*?){",
159
+ "{\\1"
160
+ ],
161
+ [
162
+ "((.्)*){",
163
+ "{\\1"
164
+ ],
165
+ [
166
+ "{",
167
+ "र्"
168
+ ],
169
+ [
170
+ "([ाीुूृेैोौंःँ]+?)(्(.्)*[^्])",
171
+ "\\2\\1"
172
+ ],
173
+ [
174
+ "्([ाीुूृेैोौंःँ]+?)((.्)*[^्])",
175
+ "्\\2\\1"
176
+ ],
177
+ [
178
+ "([ंँ])([ािीुूृेैोौः]*)",
179
+ "\\2\\1"
180
+ ],
181
+ [
182
+ "([ँंेैुू])\\1",
183
+ "\\1"
184
+ ],
185
+ [
186
+ "^ः",
187
+ ":"
188
+ ],
189
+ [
190
+ "टृ",
191
+ "ट्ट"
192
+ ],
193
+ [
194
+ "ेा",
195
+ "ाे"
196
+ ],
197
+ [
198
+ "ैा",
199
+ "ाै"
200
+ ],
201
+ [
202
+ "अाे",
203
+ "ओ"
204
+ ],
205
+ [
206
+ "अाै",
207
+ "औ"
208
+ ],
209
+ [
210
+ "अा",
211
+ "आ"
212
+ ],
213
+ [
214
+ "एे",
215
+ "ऐ"
216
+ ],
217
+ [
218
+ "ाे",
219
+ "ो"
220
+ ],
221
+ [
222
+ "ाै",
223
+ "ौ"
224
+ ]
225
+ ]
226
+ }
@@ -0,0 +1,112 @@
1
+ {
2
+ "consonants": {
3
+ "k": "क",
4
+ "kh": "ख",
5
+ "g": "ग",
6
+ "gh": "घ",
7
+ "ng": "ङ",
8
+ "ch": "च",
9
+ "chh": "छ",
10
+ "j": "ज",
11
+ "jh": "झ",
12
+ "ny": "ञ",
13
+ "T": "ट",
14
+ "Th": "ठ",
15
+ "D": "ड",
16
+ "Dh": "ढ",
17
+ "N": "ण",
18
+ "t": "त",
19
+ "th": "थ",
20
+ "d": "द",
21
+ "dh": "ध",
22
+ "n": "न",
23
+ "p": "प",
24
+ "ph": "फ",
25
+ "f": "फ",
26
+ "b": "ब",
27
+ "bh": "भ",
28
+ "m": "म",
29
+ "y": "य",
30
+ "r": "र",
31
+ "l": "ल",
32
+ "v": "व",
33
+ "w": "व",
34
+ "sh": "श",
35
+ "Sh": "ष",
36
+ "s": "स",
37
+ "h": "ह",
38
+ "ksh": "क्ष",
39
+ "gy": "ज्ञ",
40
+ "shr": "श्र",
41
+ "tr": "त्र"
42
+
43
+ },
44
+ "vowels": {
45
+ "a": "अ",
46
+ "aa": "आ",
47
+ "A": "आ",
48
+ "i": "इ",
49
+ "I": "इ",
50
+ "ee": "ई",
51
+ "ii": "ई",
52
+ "u": "उ",
53
+ "U": "उ",
54
+ "oo": "ऊ",
55
+ "uu": "ऊ",
56
+ "Ri": "ऋ",
57
+ "R": "ऋ",
58
+ "Ree": "ॠ",
59
+ "lRi": "ऌ",
60
+ "e": "ए",
61
+ "ai": "ऐ",
62
+ "o": "ओ",
63
+ "au": "औ"
64
+ },
65
+ "matras": {
66
+ "a": "",
67
+ "aa": "ा",
68
+ "A": "ा",
69
+ "i": "ि",
70
+ "I": "ि",
71
+ "ee": "ी",
72
+ "ii": "ी",
73
+ "u": "ु",
74
+ "U": "ु",
75
+ "oo": "ू",
76
+ "uu": "ू",
77
+ "Ri": "ृ",
78
+ "R": "ृ",
79
+ "Ree": "ॄ",
80
+ "e": "े",
81
+ "ai": "ै",
82
+ "o": "ो",
83
+ "au": "ौ"
84
+ },
85
+ "special": {
86
+ "om": "ॐ",
87
+ "Om": "ॐ",
88
+ "aum": "ॐ",
89
+ ".": "।",
90
+ "..": "॥",
91
+ "|": "।",
92
+ "||": "॥",
93
+ "M": "ं",
94
+ "H": "ः",
95
+ "~": "ँ"
96
+ },
97
+ "compound_consonants": {
98
+
99
+ },
100
+ "digits": {
101
+ "0": "०",
102
+ "1": "१",
103
+ "2": "२",
104
+ "3": "३",
105
+ "4": "४",
106
+ "5": "५",
107
+ "6": "६",
108
+ "7": "७",
109
+ "8": "८",
110
+ "9": "९"
111
+ }
112
+ }
@@ -0,0 +1,67 @@
1
+ {
2
+ "aum": "ॐ",
3
+ "bibek": "विवेक",
4
+ "call": "कल",
5
+ "code": "कोड",
6
+ "cool": "कूल",
7
+ "facebook": "फेसबुक",
8
+ "fun": "फन",
9
+ "garage": "ग्यारेज",
10
+ "gas": "ग्यास",
11
+ "google": "गूगल",
12
+ "hola": "होला",
13
+ "hudai": "हुँदै",
14
+ "instagram": "इन्स्टाग्राम",
15
+ "kathmandu": "काठमाडौं",
16
+ "Kathmandu": "काठमाडौं",
17
+ "laptop": "ल्याप्टप",
18
+ "message": "मेसेज",
19
+ "messenger": "मेसेन्जर",
20
+ "nepal": "नेपाल",
21
+ "nepali": "नेपाली",
22
+ "number": "नम्बर",
23
+ "pandey": "पाण्डे",
24
+ "phone": "फोन",
25
+ "plan": "प्लान",
26
+ "radio": "रेडियो",
27
+ "sms": "एसएमएस",
28
+ "SMS": "एसएमएस",
29
+ "telecom": "टेलिकम",
30
+ "twitter": "ट्विटर",
31
+ "vivek": "विवेक",
32
+ "about": "अबाउट",
33
+ "address": "ठेगाना",
34
+ "bank": "बैंक",
35
+ "battery": "ब्याट्री",
36
+ "bus": "बस",
37
+ "camera": "क्यामेरा",
38
+ "chat": "च्याट",
39
+ "city": "सिटी",
40
+ "coffee": "कफी",
41
+ "computer": "कम्प्युटर",
42
+ "contact": "कन्ट्याक्ट",
43
+ "country": "देश",
44
+ "date": "मिति",
45
+ "email": "इमेल",
46
+ "file": "फाइल",
47
+ "food": "फुड",
48
+ "friend": "फ्रेन्ड",
49
+ "game": "गेम",
50
+ "hello": "हेलो",
51
+ "internet": "इन्टरनेट",
52
+ "location": "लोकेशन",
53
+ "market": "मार्केट",
54
+ "meeting": "मिटिङ",
55
+ "mobile": "मोबाइल",
56
+ "music": "संगीत",
57
+ "name": "नाम",
58
+ "photo": "फोटो",
59
+ "school": "स्कूल",
60
+ "shop": "पसल",
61
+ "station": "स्टेसन",
62
+ "taxi": "ट्याक्सी",
63
+ "time": "समय",
64
+ "video": "भिडियो",
65
+ "wifi": "वाईफाई",
66
+ "work": "काम"
67
+ }
@@ -0,0 +1,107 @@
1
+ import re
2
+ from typing import Optional
3
+
4
+ from nepali_unicoder.loader import PreetiLoader, RuleLoader
5
+ from nepali_unicoder.tokenizer import Tokenizer
6
+ from nepali_unicoder.trie import Trie
7
+
8
+
9
+ class Engine:
10
+ VALID_MODES = {"roman", "preeti"}
11
+
12
+ def __init__(
13
+ self,
14
+ trie: Optional[Trie] = None,
15
+ tokenizer: Optional[Tokenizer] = None,
16
+ mode: str = "roman",
17
+ ):
18
+ if mode not in self.VALID_MODES:
19
+ raise ValueError(
20
+ f"Invalid mode {mode!r}. Expected one of {sorted(self.VALID_MODES)}"
21
+ )
22
+ self.mode = mode
23
+ self.post_rules = []
24
+
25
+ if trie is None:
26
+ if mode == "preeti":
27
+ loader = PreetiLoader()
28
+ # Load post-processing rules for Preeti mode
29
+ raw_rules = loader.get_post_rules()
30
+ self.post_rules = [
31
+ (re.compile(pattern), replacement)
32
+ for pattern, replacement in raw_rules
33
+ ]
34
+ else:
35
+ loader = RuleLoader()
36
+ self.trie = loader.load()
37
+ else:
38
+ self.trie = trie
39
+
40
+ if tokenizer is None:
41
+ self.tokenizer = Tokenizer()
42
+ else:
43
+ self.tokenizer = tokenizer
44
+
45
+ def transliterate(self, text: str) -> str:
46
+ """
47
+ Convert Roman text to Devanagari using the Trie.
48
+ """
49
+ if not text:
50
+ return ""
51
+
52
+ use_blocks = self.mode != "preeti"
53
+ tokens = self.tokenizer.tokenize(text, use_blocks=use_blocks)
54
+ result = []
55
+
56
+ for token in tokens:
57
+ if token.type == "BLOCK":
58
+ result.append(token.value)
59
+ elif token.type == "LITERAL":
60
+ result.append(token.value)
61
+ elif token.type == "NUMBER":
62
+ # Process Number chunk: transliterate digits, keep others (like .) as is
63
+ for char in token.value:
64
+ if char.isdigit():
65
+ # Use Trie to find digit mapping (digits are single chars in rules)
66
+ match_val, _ = self.trie.longest_match(char)
67
+ if match_val:
68
+ result.append(match_val)
69
+ else:
70
+ result.append(char)
71
+ else:
72
+ result.append(char)
73
+ elif token.type == "ROMAN":
74
+ # Process Roman chunk with Trie
75
+ chunk = token.value
76
+ idx = 0
77
+ chunk_len = len(chunk)
78
+
79
+ while idx < chunk_len:
80
+ # Try to find longest match starting at idx
81
+ # Pass chunk and idx directly to avoid slicing
82
+ match_val, match_len = self.trie.longest_match(chunk, idx)
83
+
84
+ if match_val:
85
+ result.append(match_val)
86
+ idx += match_len
87
+ else:
88
+ # No match, keep character as is
89
+ result.append(chunk[idx])
90
+ idx += 1
91
+
92
+ output = "".join(result)
93
+
94
+ # Apply post-processing rules for Preeti mode
95
+ if self.mode == "preeti" and self.post_rules:
96
+ output = self._apply_post_rules(output)
97
+
98
+ return output
99
+
100
+ def _apply_post_rules(self, text: str) -> str:
101
+ """
102
+ Apply post-processing rules (regex replacements) to the text.
103
+ Used for Preeti mode to handle contextual transformations.
104
+ """
105
+ for pattern, replacement in self.post_rules:
106
+ text = pattern.sub(replacement, text)
107
+ return text
@@ -0,0 +1,120 @@
1
+ import json
2
+ import logging
3
+ import os
4
+
5
+ from nepali_unicoder.trie import Trie
6
+
7
+ _logger = logging.getLogger(__name__)
8
+
9
+
10
+ def load_json_data(filename):
11
+ path = os.path.join(os.path.dirname(__file__), "data", filename)
12
+ try:
13
+ with open(path, "r", encoding="utf-8") as f:
14
+ return json.load(f)
15
+ except FileNotFoundError:
16
+ _logger.warning("Rule file %s not found.", filename)
17
+ return {}
18
+
19
+
20
+ class RuleLoader:
21
+ def __init__(self):
22
+ self.word_maps_path = os.path.join(
23
+ os.path.dirname(__file__), "data", "word_maps.json"
24
+ )
25
+
26
+ def load(self) -> Trie:
27
+ """Load rules and custom mappings into a Trie."""
28
+ trie = Trie()
29
+ self._load_rules(trie)
30
+ self._load_custom_mappings(trie)
31
+ return trie
32
+
33
+ def _load_rules(self, trie: Trie):
34
+ data = load_json_data("roman_rules.json")
35
+
36
+ consonants = data.get("consonants", {})
37
+ vowels = data.get("vowels", {})
38
+ matras = data.get("matras", {})
39
+ special = data.get("special", {})
40
+ digits = data.get("digits", {})
41
+
42
+ # 1. Consonants and Combinations (inserted BEFORE vowels so that
43
+ # independent vowel keys inserted next will override any collision,
44
+ # e.g. 'lRi' consonant+matra combo is overridden by vowel 'lRi' -> ऌ)
45
+ halanta = "्"
46
+
47
+ for rom_cons, dev_cons in consonants.items():
48
+ # Case 1: Consonant alone (halanta form) -> 'k' -> 'क्'
49
+ trie.add(rom_cons, dev_cons + halanta)
50
+
51
+ # Case 2: Consonant + 'a' (Schwa form) -> 'ka' -> 'क'
52
+ trie.add(rom_cons + "a", dev_cons)
53
+
54
+ # Case 3: Consonant + other vowels -> 'ki' -> 'कि'
55
+ for rom_vowel, matra in matras.items():
56
+ if rom_vowel == "a":
57
+ continue # Handled above
58
+ trie.add(rom_cons + rom_vowel, dev_cons + matra)
59
+
60
+ # 1b. Compound consonants whose romanization ends in a vowel-like letter
61
+ # (e.g. 'tra' -> 'त्र') -- these CANNOT go through the normal consonant
62
+ # loop because appending 'a' for the schwa would yield 'traa' not 'tra'.
63
+ # Instead we build their combos explicitly:
64
+ # 'tra' -> 'त्र' (natural: the trailing 'a' IS the schwa)
65
+ # 'tra' + matra -> 'त्र' + matra (e.g. 'trai' -> 'त्रि')
66
+ # 'tra' + halanta -> 'त्र्' (explicit halanta: append 'H' or use '{}')
67
+ compound_consonants = data.get("compound_consonants", {})
68
+ for rom_stem, dev_stem in compound_consonants.items():
69
+ # Schwa form: the romanization itself (e.g. 'tra' -> 'त्र')
70
+ trie.add(rom_stem, dev_stem)
71
+ # With other vowels / matras (e.g. 'trai' -> 'त्रि')
72
+ for rom_vowel, matra in matras.items():
73
+ if rom_vowel == "a":
74
+ continue # schwa already covered above
75
+ trie.add(rom_stem + rom_vowel, dev_stem + matra)
76
+
77
+ # 2. Independent Vowels (inserted AFTER consonant combos so they win
78
+ # on collision, e.g. 'lRi' -> ऌ overrides 'l'+'Ri' -> लृ)
79
+ for rom, dev in vowels.items():
80
+ trie.add(rom, dev)
81
+
82
+ # 3. Special, Digits, Punctuation
83
+ for rom, dev in special.items():
84
+ trie.add(rom, dev)
85
+
86
+ for rom, dev in digits.items():
87
+ trie.add(rom, dev)
88
+
89
+ def _load_custom_mappings(self, trie: Trie):
90
+ if not os.path.exists(self.word_maps_path):
91
+ return
92
+
93
+ with open(self.word_maps_path, "r", encoding="utf-8") as f:
94
+ mappings = json.load(f)
95
+
96
+ for roman, devanagari in mappings.items():
97
+ trie.add(roman, devanagari)
98
+ # Automatically add capitalized version if it doesn't exist
99
+ # (e.g. if 'nepal' is in maps, also add 'Nepal')
100
+ cap_roman = roman.capitalize()
101
+ if cap_roman not in mappings:
102
+ trie.add(cap_roman, devanagari)
103
+
104
+
105
+ class PreetiLoader:
106
+ def __init__(self):
107
+ # Parse the JSON file once; reused by both load() and get_post_rules()
108
+ # so the file is never read more than once per PreetiLoader instance.
109
+ self._data = load_json_data("preeti_rules.json")
110
+
111
+ def load(self) -> Trie:
112
+ """Load Preeti rules into a Trie."""
113
+ trie = Trie()
114
+ mappings = self._data.get("mappings", {})
115
+ for key, value in mappings.items():
116
+ trie.add(key, value)
117
+ return trie
118
+
119
+ def get_post_rules(self):
120
+ return self._data.get("post_rules", [])
@@ -0,0 +1,2 @@
1
+ # Rules are now loaded from JSON files in the 'data' directory.
2
+ # See src/nepali_unicoder/data/roman_rules.json and preeti_rules.json
@@ -0,0 +1,93 @@
1
+ import re
2
+ from dataclasses import dataclass
3
+ from typing import List
4
+
5
+
6
+ @dataclass
7
+ class Token:
8
+ value: str
9
+ type: str # 'ROMAN', 'BLOCK', 'PUNCTUATION', 'UNKNOWN', 'LITERAL', 'NUMBER'
10
+
11
+
12
+ class Tokenizer:
13
+ # Compiled once at class definition time -- not per tokenize() call.
14
+ _RE_BLOCK = re.compile(r"\{([^}]*)\}")
15
+ _RE_NUMBER = re.compile(r"\d*\.\d+|\d+")
16
+
17
+ def tokenize(self, text: str, use_blocks: bool = True) -> List[Token]:
18
+ """
19
+ Split text into tokens using regex-based matching for robustness.
20
+ """
21
+ tokens = []
22
+ i = 0
23
+ n = len(text)
24
+ roman_buffer = []
25
+
26
+ re_block = self._RE_BLOCK
27
+ re_number = self._RE_NUMBER
28
+
29
+ def flush_roman():
30
+ if roman_buffer:
31
+ tokens.append(Token(value="".join(roman_buffer), type="ROMAN"))
32
+ roman_buffer.clear()
33
+
34
+ while i < n:
35
+ # 1. Check for Ellipsis (Priority)
36
+ if text.startswith("...", i):
37
+ flush_roman()
38
+ tokens.append(Token(value="...", type="LITERAL"))
39
+ i += 3
40
+ continue
41
+
42
+ # 2. Check for Blocks and Braces (if enabled)
43
+ if use_blocks:
44
+ # Escaped brace {{
45
+ if text.startswith("{{", i):
46
+ flush_roman()
47
+ tokens.append(Token(value="{", type="LITERAL"))
48
+ i += 2
49
+ continue
50
+
51
+ # Block {content}
52
+ match_block = re_block.match(text, i)
53
+ if match_block:
54
+ flush_roman()
55
+ tokens.append(Token(value=match_block.group(1), type="BLOCK"))
56
+ i = match_block.end()
57
+ continue
58
+
59
+ # Escaped brace }}
60
+ if text.startswith("}}", i):
61
+ flush_roman()
62
+ tokens.append(Token(value="}", type="LITERAL"))
63
+ i += 2
64
+ continue
65
+
66
+ # Unmatched closing brace }
67
+ if text.startswith("}", i):
68
+ flush_roman()
69
+ tokens.append(Token(value="}", type="LITERAL"))
70
+ i += 1
71
+ continue
72
+
73
+ # Unmatched opening brace {
74
+ if text.startswith("{", i):
75
+ flush_roman()
76
+ tokens.append(Token(value="{", type="LITERAL"))
77
+ i += 1
78
+ continue
79
+
80
+ # 3. Check for Numbers
81
+ match_number = re_number.match(text, i)
82
+ if match_number:
83
+ flush_roman()
84
+ tokens.append(Token(value=match_number.group(0), type="NUMBER"))
85
+ i = match_number.end()
86
+ continue
87
+
88
+ # 4. Fallback: Accumulate Roman
89
+ roman_buffer.append(text[i])
90
+ i += 1
91
+
92
+ flush_roman()
93
+ return tokens
@@ -0,0 +1,53 @@
1
+ from typing import Dict, Optional, Tuple
2
+
3
+
4
+ class TrieNode:
5
+ __slots__ = ("children", "value", "is_end")
6
+
7
+ def __init__(self):
8
+ self.children: Dict[str, TrieNode] = {}
9
+ self.value: Optional[str] = None
10
+ self.is_end: bool = False
11
+
12
+
13
+ class Trie:
14
+ def __init__(self):
15
+ self.root = TrieNode()
16
+
17
+ def add(self, key: str, value: str) -> None:
18
+ """Insert a key-value pair into the Trie."""
19
+ node = self.root
20
+ for char in key:
21
+ if char not in node.children:
22
+ node.children[char] = TrieNode()
23
+ node = node.children[char]
24
+ node.is_end = True
25
+ node.value = value
26
+
27
+ def longest_match(
28
+ self, text: str, start_index: int = 0
29
+ ) -> Tuple[Optional[str], int]:
30
+ """
31
+ Find the longest key in the Trie that matches the start of the text.
32
+ Returns (value, length_of_match).
33
+ If no match found, returns (None, 0).
34
+ """
35
+ node = self.root
36
+ last_match_value = None
37
+ last_match_len = 0
38
+ current_len = 0
39
+ n = len(text)
40
+ i = start_index
41
+
42
+ while i < n:
43
+ char = text[i]
44
+ if char not in node.children:
45
+ break
46
+ node = node.children[char]
47
+ current_len += 1
48
+ i += 1
49
+ if node.is_end:
50
+ last_match_value = node.value
51
+ last_match_len = current_len
52
+
53
+ return last_match_value, last_match_len
@@ -0,0 +1,210 @@
1
+ Metadata-Version: 2.4
2
+ Name: nepali_unicoder
3
+ Version: 0.2.0
4
+ Summary: A Python package for Roman to Nepali (Devanagari) transliteration
5
+ Author-email: realsanjeev <075bei033.sanjeev@pcampus.edu.np>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/realsanjeev/nepali_unicoder
8
+ Project-URL: Documentation, https://realsanjeev.github.io/nepali_unicoder/
9
+ Project-URL: Bug Tracker, https://github.com/realsanjeev/nepali_unicoder/issues
10
+ Keywords: nepali,unicoder,roman to devanagari,nepali transliteration,preeti to unicode
11
+ Classifier: Programming Language :: Python
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Operating System :: OS Independent
14
+ Requires-Python: >=3.7
15
+ Description-Content-Type: text/markdown
16
+ License-File: LICENSE
17
+ Provides-Extra: dev
18
+ Requires-Dist: pre-commit; extra == "dev"
19
+ Requires-Dist: mkdocs; extra == "dev"
20
+ Requires-Dist: mkdocs-material; extra == "dev"
21
+ Dynamic: license-file
22
+
23
+ # Nepali Unicoder
24
+
25
+ [![Documentation](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://realsanjeev.github.io/nepali_unicoder/)
26
+ [![PyPI version](https://badge.fury.io/py/nepali-unicoder.svg)](https://pypi.org/project/nepali-unicoder/)
27
+
28
+ A robust Python package for converting Romanized Nepali text and Preeti font text into Unicode Devanagari script. It uses a greedy matching algorithm for Roman transliteration and a two-phase conversion process for Preeti with contextual rules.
29
+
30
+ Read the **[Full Documentation](https://realsanjeev.github.io/nepali_unicoder/)** for detailed usage guides, Preeti mapping references, and API details.
31
+
32
+ ## Features
33
+
34
+ - **Accurate Transliteration**: Uses a greedy matching algorithm to prioritize longer phonetic matches (e.g., 'kha' is matched before 'k' and 'h').
35
+ - **Preeti Font Support**: Full support for Preeti to Unicode conversion with 30+ contextual rules for accurate transformation.
36
+ - **Smart Vowel Handling**: Distinguishes between independent vowels (e.g., 'aa' -> 'आ') and vowel signs/matras (e.g., 'ka' -> 'क', 'kaa' -> 'का').
37
+ - **Contextual Rules**: Handles complex Devanagari rules like reph positioning, matra reordering, and special character combinations.
38
+ - **Mixed Content Support**: Allows keeping English words or specific text in Roman script using `{}` blocks.
39
+ - **Customizable**: Supports custom word-level overrides via `word_maps.json`.
40
+ - **CLI Support**: Can be used directly from the command line.
41
+
42
+ ## Installation
43
+
44
+ You can install the package locally:
45
+
46
+ ```bash
47
+ pip install nepali-unicoder
48
+ ```
49
+
50
+ ## Usage
51
+
52
+ ### Command Line Interface (CLI)
53
+
54
+ You can use the converter directly from the terminal:
55
+
56
+ ```bash
57
+ # Using the installed command
58
+ nepali-unicoder "namaste"
59
+ # Output: नमस्ते
60
+
61
+ # Or using python module
62
+ python -m nepali_unicoder "namaste"
63
+ # Output: नमस्ते
64
+
65
+ # Pipe input
66
+ echo "mero naam sanjeev ho" | nepali-unicoder
67
+ # Output: मेरो नाम् सन्जीव् हो
68
+ ```
69
+
70
+ ### Python API
71
+
72
+ ```python
73
+ from nepali_unicoder.convert import Converter
74
+
75
+ converter = Converter()
76
+
77
+ # Basic conversion
78
+ text = "namaste nepal"
79
+ print(converter.convert(text))
80
+ # Output: नमस्ते नेपाल
81
+
82
+ # Using 'as-is' blocks for English text
83
+ mixed_text = "mero naam {Sanjeev} ho"
84
+ print(converter.convert(mixed_text))
85
+ # Output: मेरो नाम् Sanjeev हो
86
+ ```
87
+
88
+ ### Preeti Mode
89
+
90
+ Convert Preeti font text to Unicode with full support for contextual rules:
91
+
92
+ ```python
93
+ from nepali_unicoder.convert import Converter
94
+
95
+ # Create converter in Preeti mode
96
+ preeti_converter = Converter(mode="preeti")
97
+
98
+ # Basic conversion
99
+ preeti_text = "s{sf" # Preeti characters
100
+ print(preeti_converter.convert(preeti_text))
101
+ # Output: र्कका
102
+
103
+ # The converter handles:
104
+ # - Reph positioning: { → र् (moves before consonant)
105
+ # - Matra reordering: l (ि) moves after consonant
106
+ # - Special m transformations
107
+ # - Vowel combinations
108
+ # - Literal brackets: { and } are treated as normal characters in Preeti mode
109
+ ```
110
+
111
+ #### Preeti Character Examples
112
+
113
+ | Preeti | Unicode | Description |
114
+ |--------|---------|-------------|
115
+ | `s` | `क` | Consonant ka |
116
+ | `s{` | `र्क` | Reph + ka (contextual) |
117
+ | `sl` | `कि` | ka + short i (reordered) |
118
+ | `qm` | `क्र` | Special m transformation |
119
+ | `!@#` | `१२३` | Nepali numbers |
120
+ | `Ù` / `Ú` | `;` / `:` | Literal punctuation |
121
+ | `«` / `»` | `्र` | Ra-foot (for ट, ठ, ड, ढ) |
122
+ | `¿` | `रू` | Combined ruu |
123
+ | `å` | `द्व` | Combined dva |
124
+ | `ˆ` | `फ्` | Half ph |
125
+ | `ª` | `ङ` | Consonant nga |
126
+ | `æ` / `Æ` | `“` / `”` | Curly quotes |
127
+ | `¥` | `र्` | Half ra |
128
+ | `¶` | `ठ्ठ` | Combined thth |
129
+ | `§` | `ट्ट` | Combined tt |
130
+ | `£` | `घ्` | Half gh |
131
+ | `Ë` / `Í` | `ङ्ग` / `ङ्क` | Combined nga-ga / nga-ka |
132
+ | `‰` | `झ्` | Half jh |
133
+
134
+ #### CLI for Preeti
135
+
136
+ ```bash
137
+ python -m nepali_unicoder --preeti "s{sf"
138
+ # Output: र्कका
139
+ ```
140
+
141
+
142
+ ## Transliteration Rules
143
+
144
+ - **Consonants**: `k` -> `क्`, `ka` -> `क`, `kh` -> `ख्`, `kha` -> `ख`
145
+ - **Vowels**: `a` -> `अ`, `aa` -> `आ`, `i` -> `इ`, `u` -> `उ`
146
+ - **Matras**: `ki` -> `कि`, `ko` -> `को`
147
+ - **Special**: `.` -> `।`, `..` -> `॥`
148
+ - **Numbers**: `0-9` -> `०-९` (Decimal points are preserved: `1.5` -> `१.५`)
149
+
150
+ ## Advanced Usage
151
+
152
+ ### Handling Complex Text
153
+
154
+ The converter handles mixed content gracefully. You can use `{}` to keep text as-is (e.g., for English words or code snippets).
155
+
156
+ ```python
157
+ text = "mero naam {Sanjeev} ho ra ma 12.5 barsa ko bhaye."
158
+ print(converter.convert(text))
159
+ # Output: मेरो नाम् Sanjeev हो र म १२.५ बर्स को भये।
160
+ ```
161
+
162
+ ### Configuration
163
+
164
+ The package uses `word_maps.json` for custom word-level overrides, located in the `src/nepali_unicoder` directory.
165
+
166
+ 1. **`word_maps.json`**: Defines custom word-level overrides. Use this for words that don't follow standard phonetic rules.
167
+
168
+ Example `word_maps.json`:
169
+ ```json
170
+ {
171
+ "nepal": "नेपाल",
172
+ "kathamandu": "काठमाडौँ"
173
+ }
174
+ ```
175
+
176
+ ## Contribution
177
+
178
+ We welcome contributions! Here's how you can help:
179
+
180
+ 1. **Clone the repository**:
181
+ ```bash
182
+ git clone https://github.com/realsanjeev/nepali_unicoder.git
183
+ cd nepali_unicoder
184
+ ```
185
+
186
+ 2. **Set up a virtual environment**:
187
+ ```bash
188
+ python3 -m venv .venv
189
+ source .venv/bin/activate
190
+ pip install -e .
191
+ ```
192
+
193
+ 3. **Run tests**:
194
+ ```bash
195
+ python -m unittest discover tests
196
+ ```
197
+
198
+ 4. **Submit a Pull Request**: Create a new branch, make your changes, and submit a PR.
199
+
200
+ ## Development
201
+
202
+ To run tests:
203
+
204
+ ```bash
205
+ python -m unittest discover tests
206
+ ```
207
+
208
+ ## License
209
+
210
+ [MIT](./LICENSE)
@@ -0,0 +1,17 @@
1
+ nepali_unicoder/__init__.py,sha256=ll1McRUqVExqDwXUNpMhRvfzMYKp0DPevUjKofmgoec,71
2
+ nepali_unicoder/__main__.py,sha256=dsrU6FZQM-4oORWuzteKE897f8Ahr3bg9DmPhg6AzfE,974
3
+ nepali_unicoder/convert.py,sha256=st0O0zc_-UZiGjcHCo8RJUNkjjam1cmecs39VT4zlZ0,306
4
+ nepali_unicoder/engine.py,sha256=hKSI58hOt7Ek5Q98CWBbOHbQm0jPa-solWySVuevvEk,3651
5
+ nepali_unicoder/loader.py,sha256=Qx9l2QkbeDg-0sDuIW_PJcf6-5iaFxn52NyTAOPqK-E,4593
6
+ nepali_unicoder/rules.py,sha256=Ezxpwz0bq55QWzr_qU9HR-spEa5W3zy-xYolqoLPVGE,134
7
+ nepali_unicoder/tokenizer.py,sha256=XtygIM1Pgz908MDQmycevGG80XVfgT_wyPN9v-_cnSI,2934
8
+ nepali_unicoder/trie.py,sha256=7HF1WCSaCUP0x3uEIiXxdyyuHDI_j2B1nbnm976P7Wo,1478
9
+ nepali_unicoder/data/preeti_rules.json,sha256=_0-UXtDY7_mHnzRsUWibh3FU4S448Crw25ImrT_p9YQ,4426
10
+ nepali_unicoder/data/roman_rules.json,sha256=F8FuDfYvA17CpVjgOCrRuev2bDyViT6MA-qaWi1pKlo,2147
11
+ nepali_unicoder/data/word_maps.json,sha256=z6di4YZn_EGvkKvNa9bffLcYw-esTAKYA2PJPIqBqrk,2158
12
+ nepali_unicoder-0.2.0.dist-info/licenses/LICENSE,sha256=nWolli2fvrHF2ActODUDH0QjSKZf8EDb-M-vp74xa9I,1068
13
+ nepali_unicoder-0.2.0.dist-info/METADATA,sha256=re8lsooBQwdjIKn_wgnZSbOlLhVXT6PiHweLDOiDFFM,6626
14
+ nepali_unicoder-0.2.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
15
+ nepali_unicoder-0.2.0.dist-info/entry_points.txt,sha256=JtHpymd3LbtuCj1uPl9p2NsACU_fdMSSjdYtJ-VUZGU,66
16
+ nepali_unicoder-0.2.0.dist-info/top_level.txt,sha256=hoBnas3ztt0HTE0NM5jWgBsqDetj9OPg4rMwMPTqM3I,16
17
+ nepali_unicoder-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (82.0.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ nepali-unicoder = nepali_unicoder.__main__:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 RealSanjeev
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ nepali_unicoder