tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/tier4.py

@@ -0,0 +1,410 @@
+"""
+Tier 4 Languages - Extended Global Coverage
+============================================
+
+25+ languages: South Asian, Southeast Asian, African, Caucasian, Central Asian, etc.
+bn, ta, te, mr, gu, kn, ml, pa, ur, ne, si, my, km, lo, tl, sw, am, yo, ha, zu, af, ka, az, kk, uz, mn
+"""
+
+from .templates.latin_template import LatinScriptAnalyzer
+from .templates.cyrillic_template import CyrillicScriptAnalyzer
+from .templates.arabic_script_template import ArabicScriptAnalyzer
+from .templates.brahmic_template import BrahmicScriptAnalyzer
+from .templates.other_scripts_template import GeorgianScriptAnalyzer, EthiopicScriptAnalyzer
+
+
+# =============================================================================
+# South Asian Languages
+# =============================================================================
+
+class BengaliAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "bn"
+    LANG_NAME = "Bengali"
+    SCRIPT_NAME = "bengali"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'এ': 'PSP', 'তে': 'PSP', 'কে': 'PSP', 'র': 'PSP', 'থেকে': 'PSP'}
+        self.function_words = {
+            'আমি': 'PRON', 'তুমি': 'PRON', 'সে': 'PRON', 'আমরা': 'PRON', 'তোমরা': 'PRON', 'তারা': 'PRON',
+            'এবং': 'CONJ', 'বা': 'CONJ', 'কিন্তু': 'CONJ', 'যে': 'CONJ',
+            'না': 'NEG', 'হ্যাঁ': 'ADV', 'খুব': 'ADV', 'ভালো': 'ADV',
+        }
+
+
+class TamilAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "ta"
+    LANG_NAME = "Tamil"
+    SCRIPT_NAME = "tamil"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'இல்': 'PSP', 'இருந்து': 'PSP', 'க்கு': 'PSP', 'உடன்': 'PSP'}
+        self.function_words = {
+            'நான்': 'PRON', 'நீ': 'PRON', 'அவன்': 'PRON', 'அவள்': 'PRON', 'நாங்கள்': 'PRON', 'அவர்கள்': 'PRON',
+            'மற்றும்': 'CONJ', 'அல்லது': 'CONJ', 'ஆனால்': 'CONJ',
+            'இல்லை': 'NEG', 'ஆம்': 'ADV', 'மிகவும்': 'ADV', 'நன்றாக': 'ADV',
+        }
+
+
+class TeluguAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "te"
+    LANG_NAME = "Telugu"
+    SCRIPT_NAME = "telugu"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'లో': 'PSP', 'కి': 'PSP', 'నుండి': 'PSP', 'తో': 'PSP'}
+        self.function_words = {
+            'నేను': 'PRON', 'నీవు': 'PRON', 'అతను': 'PRON', 'ఆమె': 'PRON', 'మేము': 'PRON', 'వారు': 'PRON',
+            'మరియు': 'CONJ', 'లేదా': 'CONJ', 'కానీ': 'CONJ',
+            'కాదు': 'NEG', 'అవును': 'ADV', 'చాలా': 'ADV', 'బాగా': 'ADV',
+        }
+
+
+class MarathiAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "mr"
+    LANG_NAME = "Marathi"
+    SCRIPT_NAME = "devanagari"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'मध्ये': 'PSP', 'ला': 'PSP', 'पासून': 'PSP', 'साठी': 'PSP', 'बरोबर': 'PSP'}
+        self.function_words = {
+            'मी': 'PRON', 'तू': 'PRON', 'तो': 'PRON', 'ती': 'PRON', 'आम्ही': 'PRON', 'ते': 'PRON',
+            'आणि': 'CONJ', 'किंवा': 'CONJ', 'पण': 'CONJ',
+            'नाही': 'NEG', 'हो': 'ADV', 'खूप': 'ADV', 'चांगले': 'ADV',
+        }
+
+
+class GujaratiAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "gu"
+    LANG_NAME = "Gujarati"
+    SCRIPT_NAME = "gujarati"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'માં': 'PSP', 'ને': 'PSP', 'થી': 'PSP', 'માટે': 'PSP'}
+        self.function_words = {
+            'હું': 'PRON', 'તું': 'PRON', 'તે': 'PRON', 'અમે': 'PRON', 'તેઓ': 'PRON',
+            'અને': 'CONJ', 'અથવા': 'CONJ', 'પણ': 'CONJ',
+            'નહીં': 'NEG', 'હા': 'ADV', 'ખૂબ': 'ADV', 'સારું': 'ADV',
+        }
+
+
+class KannadaAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "kn"
+    LANG_NAME = "Kannada"
+    SCRIPT_NAME = "kannada"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'ಲ್ಲಿ': 'PSP', 'ಗೆ': 'PSP', 'ಇಂದ': 'PSP', 'ಜೊತೆ': 'PSP'}
+        self.function_words = {
+            'ನಾನು': 'PRON', 'ನೀನು': 'PRON', 'ಅವನು': 'PRON', 'ಅವಳು': 'PRON', 'ನಾವು': 'PRON', 'ಅವರು': 'PRON',
+            'ಮತ್ತು': 'CONJ', 'ಅಥವಾ': 'CONJ', 'ಆದರೆ': 'CONJ',
+            'ಇಲ್ಲ': 'NEG', 'ಹೌದು': 'ADV', 'ತುಂಬಾ': 'ADV', 'ಚೆನ್ನಾಗಿ': 'ADV',
+        }
+
+
+class MalayalamAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "ml"
+    LANG_NAME = "Malayalam"
+    SCRIPT_NAME = "malayalam"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'ൽ': 'PSP', 'ക്ക്': 'PSP', 'ൽ നിന്ന്': 'PSP', 'കൂടെ': 'PSP'}
+        self.function_words = {
+            'ഞാൻ': 'PRON', 'നീ': 'PRON', 'അവൻ': 'PRON', 'അവൾ': 'PRON', 'ഞങ്ങൾ': 'PRON', 'അവർ': 'PRON',
+            'ഒപ്പം': 'CONJ', 'അല്ലെങ്കിൽ': 'CONJ', 'പക്ഷേ': 'CONJ',
+            'ഇല്ല': 'NEG', 'അതെ': 'ADV', 'വളരെ': 'ADV', 'നന്നായി': 'ADV',
+        }
+
+
+class PunjabiAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "pa"
+    LANG_NAME = "Punjabi"
+    SCRIPT_NAME = "gurmukhi"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'ਵਿੱਚ': 'PSP', 'ਨੂੰ': 'PSP', 'ਤੋਂ': 'PSP', 'ਲਈ': 'PSP', 'ਨਾਲ': 'PSP'}
+        self.function_words = {
+            'ਮੈਂ': 'PRON', 'ਤੂੰ': 'PRON', 'ਉਹ': 'PRON', 'ਅਸੀਂ': 'PRON', 'ਉਹ': 'PRON',
+            'ਅਤੇ': 'CONJ', 'ਜਾਂ': 'CONJ', 'ਪਰ': 'CONJ',
+            'ਨਹੀਂ': 'NEG', 'ਹਾਂ': 'ADV', 'ਬਹੁਤ': 'ADV', 'ਚੰਗੀ': 'ADV',
+        }
+
+
+class UrduAnalyzer(ArabicScriptAnalyzer):
+    LANG_CODE = "ur"
+    LANG_NAME = "Urdu"
+
+    def _build_base_dictionary(self):
+        self.prefixes = {}
+        self.suffixes = {'وں': 'PL', 'یں': 'PL'}
+        self.function_words = {
+            'میں': 'PRON', 'تم': 'PRON', 'وہ': 'PRON', 'ہم': 'PRON', 'آپ': 'PRON',
+            'میں': 'PREP', 'پر': 'PREP', 'سے': 'PREP', 'کو': 'PREP', 'کے': 'PREP',
+            'اور': 'CONJ', 'یا': 'CONJ', 'لیکن': 'CONJ', 'کہ': 'CONJ',
+            'نہیں': 'NEG', 'ہاں': 'ADV', 'بہت': 'ADV', 'اچھا': 'ADV',
+        }
+
+
+class NepaliAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "ne"
+    LANG_NAME = "Nepali"
+    SCRIPT_NAME = "devanagari"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'मा': 'PSP', 'लाई': 'PSP', 'बाट': 'PSP', 'को': 'PSP', 'सँग': 'PSP'}
+        self.function_words = {
+            'म': 'PRON', 'तिमी': 'PRON', 'उ': 'PRON', 'हामी': 'PRON', 'उनीहरू': 'PRON',
+            'र': 'CONJ', 'वा': 'CONJ', 'तर': 'CONJ',
+            'होइन': 'NEG', 'हो': 'ADV', 'धेरै': 'ADV', 'राम्रो': 'ADV',
+        }
+
+
+class SinhalaAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "si"
+    LANG_NAME = "Sinhala"
+    SCRIPT_NAME = "sinhala"
+
+    def _build_base_dictionary(self):
+        self.postpositions = {'ට': 'PSP', 'ගෙන්': 'PSP', 'සමග': 'PSP'}
+        self.function_words = {
+            'මම': 'PRON', 'ඔබ': 'PRON', 'ඔහු': 'PRON', 'ඇය': 'PRON', 'අපි': 'PRON', 'ඔවුන්': 'PRON',
+            'සහ': 'CONJ', 'හෝ': 'CONJ', 'නමුත්': 'CONJ',
+            'නැත': 'NEG', 'ඔව්': 'ADV', 'ඉතා': 'ADV', 'හොඳ': 'ADV',
+        }
+
+
+# =============================================================================
+# Southeast Asian Languages
+# =============================================================================
+
+class MyanmarAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "my"
+    LANG_NAME = "Myanmar/Burmese"
+    SCRIPT_NAME = "myanmar"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'ကျွန်တော်': 'PRON', 'သင်': 'PRON', 'သူ': 'PRON', 'ကျွန်တော်တို့': 'PRON', 'သူတို့': 'PRON',
+            'နှင့်': 'CONJ', 'သို့မဟုတ်': 'CONJ', 'သို့သော်': 'CONJ',
+            'မဟုတ်': 'NEG', 'ဟုတ်': 'ADV', 'အလွန်': 'ADV', 'ကောင်း': 'ADV',
+        }
+
+
+class KhmerAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "km"
+    LANG_NAME = "Khmer"
+    SCRIPT_NAME = "khmer"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'ខ្ញុំ': 'PRON', 'អ្នក': 'PRON', 'គាត់': 'PRON', 'យើង': 'PRON', 'ពួកគេ': 'PRON',
+            'និង': 'CONJ', 'ឬ': 'CONJ', 'ប៉ុន្តែ': 'CONJ',
+            'មិន': 'NEG', 'បាទ': 'ADV', 'ខ្លាំង': 'ADV', 'ល្អ': 'ADV',
+        }
+
+
+class LaoAnalyzer(BrahmicScriptAnalyzer):
+    LANG_CODE = "lo"
+    LANG_NAME = "Lao"
+    SCRIPT_NAME = "lao"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'ຂ້ອຍ': 'PRON', 'ເຈົ້າ': 'PRON', 'ລາວ': 'PRON', 'ພວກເຮົາ': 'PRON', 'ພວກເຂົາ': 'PRON',
+            'ແລະ': 'CONJ', 'ຫຼື': 'CONJ', 'ແຕ່': 'CONJ',
+            'ບໍ່': 'NEG', 'ແມ່ນ': 'ADV', 'ຫຼາຍ': 'ADV', 'ດີ': 'ADV',
+        }
+
+
+class TagalogAnalyzer(LatinScriptAnalyzer):
+    LANG_CODE = "tl"
+    LANG_NAME = "Tagalog/Filipino"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'ang': 'DET', 'ng': 'DET', 'mga': 'DET',
+            'ako': 'PRON', 'ikaw': 'PRON', 'siya': 'PRON', 'kami': 'PRON', 'tayo': 'PRON', 'sila': 'PRON',
+            'sa': 'PREP', 'para': 'PREP', 'mula': 'PREP',
+            'at': 'CONJ', 'o': 'CONJ', 'pero': 'CONJ', 'na': 'CONJ',
+            'hindi': 'NEG', 'oo': 'ADV', 'napaka': 'ADV', 'mabuti': 'ADV',
+        }
+
+
+class MalayAnalyzer(LatinScriptAnalyzer):
+    LANG_CODE = "ms"
+    LANG_NAME = "Malay"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'ini': 'DET', 'itu': 'DET', 'tersebut': 'DET', 'semua': 'DET', 'setiap': 'DET',
+            'saya': 'PRON', 'aku': 'PRON', 'awak': 'PRON', 'anda': 'PRON', 'dia': 'PRON', 'ia': 'PRON',
+            'kami': 'PRON', 'kita': 'PRON', 'mereka': 'PRON', 'beliau': 'PRON',
+            'di': 'PREP', 'ke': 'PREP', 'dari': 'PREP', 'daripada': 'PREP', 'kepada': 'PREP',
+            'pada': 'PREP', 'untuk': 'PREP', 'dengan': 'PREP', 'oleh': 'PREP', 'dalam': 'PREP',
+            'dan': 'CONJ', 'atau': 'CONJ', 'tetapi': 'CONJ', 'namun': 'CONJ', 'kerana': 'CONJ',
+            'jika': 'CONJ', 'kalau': 'CONJ', 'apabila': 'CONJ', 'bahawa': 'CONJ', 'supaya': 'CONJ',
+            'tidak': 'NEG', 'bukan': 'NEG', 'belum': 'NEG',
+            'adalah': 'AUX', 'ialah': 'AUX', 'ada': 'AUX', 'akan': 'AUX', 'sudah': 'AUX',
+            'telah': 'AUX', 'sedang': 'AUX', 'masih': 'AUX', 'boleh': 'AUX', 'dapat': 'AUX',
+            'sangat': 'ADV', 'amat': 'ADV', 'lebih': 'ADV', 'paling': 'ADV', 'juga': 'ADV',
+        }
+
+
+# =============================================================================
+# African Languages
+# =============================================================================
+
+class SwahiliAnalyzer(LatinScriptAnalyzer):
+    LANG_CODE = "sw"
+    LANG_NAME = "Swahili"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'mimi': 'PRON', 'wewe': 'PRON', 'yeye': 'PRON', 'sisi': 'PRON', 'ninyi': 'PRON', 'wao': 'PRON',
+            'na': 'CONJ', 'au': 'CONJ', 'lakini': 'CONJ', 'kwa': 'PREP',
+            'hapana': 'NEG', 'ndiyo': 'ADV', 'sana': 'ADV', 'vizuri': 'ADV',
+        }
+
+
+class AmharicAnalyzer(EthiopicScriptAnalyzer):
+    pass  # Uses template
+
+
+class YorubaAnalyzer(LatinScriptAnalyzer):
+    LANG_CODE = "yo"
+    LANG_NAME = "Yoruba"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'èmi': 'PRON', 'ìwọ': 'PRON', 'òun': 'PRON', 'àwa': 'PRON', 'wọ́n': 'PRON',
+            'àti': 'CONJ', 'tàbí': 'CONJ', 'ṣùgbọ́n': 'CONJ',
+            'kò': 'NEG', 'bẹ́ẹ̀ni': 'ADV', 'púpọ̀': 'ADV', 'dáradára': 'ADV',
+        }
+
+
+class HausaAnalyzer(LatinScriptAnalyzer):
+    LANG_CODE = "ha"
+    LANG_NAME = "Hausa"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'ni': 'PRON', 'kai': 'PRON', 'shi': 'PRON', 'ita': 'PRON', 'mu': 'PRON', 'su': 'PRON',
+            'da': 'CONJ', 'ko': 'CONJ', 'amma': 'CONJ',
+            'ba': 'NEG', 'e': 'ADV', 'sosai': 'ADV', 'da kyau': 'ADV',
+        }
+
+
+class ZuluAnalyzer(LatinScriptAnalyzer):
+    LANG_CODE = "zu"
+    LANG_NAME = "Zulu"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'mina': 'PRON', 'wena': 'PRON', 'yena': 'PRON', 'thina': 'PRON', 'bona': 'PRON',
+            'na': 'CONJ', 'noma': 'CONJ', 'kodwa': 'CONJ',
+            'cha': 'NEG', 'yebo': 'ADV', 'kakhulu': 'ADV', 'kahle': 'ADV',
+        }
+
+
+class AfrikaansAnalyzer(LatinScriptAnalyzer):
+    LANG_CODE = "af"
+    LANG_NAME = "Afrikaans"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'die': 'DET', "'n": 'DET',
+            'ek': 'PRON', 'jy': 'PRON', 'hy': 'PRON', 'sy': 'PRON', 'ons': 'PRON', 'hulle': 'PRON',
+            'in': 'PREP', 'op': 'PREP', 'van': 'PREP', 'met': 'PREP', 'vir': 'PREP',
+            'en': 'CONJ', 'of': 'CONJ', 'maar': 'CONJ', 'dat': 'CONJ',
+            'nie': 'NEG', 'ja': 'ADV', 'baie': 'ADV', 'goed': 'ADV',
+        }
+
+
+# =============================================================================
+# Caucasian Languages
+# =============================================================================
+
+class GeorgianAnalyzer(GeorgianScriptAnalyzer):
+    pass  # Uses template
+
+
+class ArmenianAnalyzer(LatinScriptAnalyzer):
+    """Armenian with Latin transliteration support"""
+    LANG_CODE = "hy"
+    LANG_NAME = "Armenian"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'yes': 'PRON', 'duk': 'PRON', 'na': 'PRON', 'menk': 'PRON', 'nrank': 'PRON',
+            'yev': 'CONJ', 'kam': 'CONJ', 'bayc': 'CONJ',
+            'che': 'NEG', 'ayo': 'ADV', 'shat': 'ADV', 'lav': 'ADV',
+        }
+
+
+class AzerbaijaniAnalyzer(LatinScriptAnalyzer):
+    LANG_CODE = "az"
+    LANG_NAME = "Azerbaijani"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'mən': 'PRON', 'sən': 'PRON', 'o': 'PRON', 'biz': 'PRON', 'siz': 'PRON', 'onlar': 'PRON',
+            'və': 'CONJ', 'və ya': 'CONJ', 'amma': 'CONJ', 'lakin': 'CONJ',
+            'yox': 'NEG', 'bəli': 'ADV', 'çox': 'ADV', 'yaxşı': 'ADV',
+        }
+
+
+# =============================================================================
+# Central Asian Languages
+# =============================================================================
+
+class KazakhAnalyzer(CyrillicScriptAnalyzer):
+    LANG_CODE = "kk"
+    LANG_NAME = "Kazakh"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'мен': 'PRON', 'сен': 'PRON', 'ол': 'PRON', 'біз': 'PRON', 'сіз': 'PRON', 'олар': 'PRON',
+            'және': 'CONJ', 'немесе': 'CONJ', 'бірақ': 'CONJ',
+            'жоқ': 'NEG', 'иә': 'ADV', 'өте': 'ADV', 'жақсы': 'ADV',
+        }
+
+
+class UzbekAnalyzer(LatinScriptAnalyzer):
+    LANG_CODE = "uz"
+    LANG_NAME = "Uzbek"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'men': 'PRON', 'sen': 'PRON', 'u': 'PRON', 'biz': 'PRON', 'siz': 'PRON', 'ular': 'PRON',
+            'va': 'CONJ', 'yoki': 'CONJ', 'lekin': 'CONJ', 'ammo': 'CONJ',
+            "yo'q": 'NEG', 'ha': 'ADV', 'juda': 'ADV', 'yaxshi': 'ADV',
+        }
+
+
+class MongolianAnalyzer(CyrillicScriptAnalyzer):
+    LANG_CODE = "mn"
+    LANG_NAME = "Mongolian"
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'би': 'PRON', 'чи': 'PRON', 'тэр': 'PRON', 'бид': 'PRON', 'та': 'PRON', 'тэд': 'PRON',
+            'ба': 'CONJ', 'эсвэл': 'CONJ', 'гэхдээ': 'CONJ',
+            'үгүй': 'NEG', 'тийм': 'ADV', 'маш': 'ADV', 'сайн': 'ADV',
+        }
+
+
+# Export all
+__all__ = [
+    # South Asian
+    'BengaliAnalyzer', 'TamilAnalyzer', 'TeluguAnalyzer', 'MarathiAnalyzer',
+    'GujaratiAnalyzer', 'KannadaAnalyzer', 'MalayalamAnalyzer', 'PunjabiAnalyzer',
+    'UrduAnalyzer', 'NepaliAnalyzer', 'SinhalaAnalyzer',
+    # Southeast Asian
+    'MyanmarAnalyzer', 'KhmerAnalyzer', 'LaoAnalyzer', 'TagalogAnalyzer', 'MalayAnalyzer',
+    # African
+    'SwahiliAnalyzer', 'AmharicAnalyzer', 'YorubaAnalyzer', 'HausaAnalyzer',
+    'ZuluAnalyzer', 'AfrikaansAnalyzer',
+    # Caucasian
+    'GeorgianAnalyzer', 'ArmenianAnalyzer', 'AzerbaijaniAnalyzer',
+    # Central Asian
+    'KazakhAnalyzer', 'UzbekAnalyzer', 'MongolianAnalyzer',
+]