tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/templates/other_scripts_template.py

@@ -0,0 +1,475 @@
+"""
+Other Script Language Templates
+===============================
+
+기타 문자 체계용 템플릿 분석기
+Hebrew, Greek, Georgian, Armenian, Thai, Ethiopic, etc.
+"""
+
+import re
+from typing import List, Tuple, Dict, Optional
+
+from ..advanced_base import (
+    AdvancedMorphologicalAnalyzer, Morpheme, AnalysisResult, Domain
+)
+
+
+class HebrewScriptAnalyzer(AdvancedMorphologicalAnalyzer):
+    """히브리 문자 기반 언어 템플릿 (Hebrew, Yiddish)"""
+
+    LANG_CODE = "he"
+    LANG_NAME = "Hebrew"
+
+    WORD_PATTERN = re.compile(r'[\u0590-\u05FF\uFB1D-\uFB4F]+')
+    NUMBER_PATTERN = re.compile(r'[0-9]+')
+
+    def __init__(self):
+        super().__init__()
+
+    def _build_base_dictionary(self):
+        self.prefixes = {'ה': 'DEF', 'ו': 'CONJ', 'ב': 'PREP', 'ל': 'PREP', 'מ': 'PREP', 'כ': 'PREP'}
+        self.function_words = {
+            'אני': 'PRON', 'אתה': 'PRON', 'את': 'PRON', 'הוא': 'PRON', 'היא': 'PRON',
+            'אנחנו': 'PRON', 'אתם': 'PRON', 'הם': 'PRON', 'הן': 'PRON',
+            'של': 'PREP', 'על': 'PREP', 'עם': 'PREP', 'אל': 'PREP',
+            'לא': 'NEG', 'אין': 'NEG', 'כן': 'ADV', 'גם': 'ADV',
+        }
+
+    def _build_domain_dictionaries(self):
+        pass
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        if not text.strip():
+            return [AnalysisResult([])]
+        morphemes = self._analyze_text(text, domain)
+        result = AnalysisResult(morphemes=morphemes, score=1.0, domain=domain)
+        return [result]
+
+    def _analyze_text(self, text: str, domain: Domain) -> List[Morpheme]:
+        result = []
+        pos = 0
+        while pos < len(text):
+            if text[pos].isspace():
+                pos += 1
+                continue
+            word_match = self.WORD_PATTERN.match(text[pos:])
+            if word_match:
+                word = word_match.group()
+                morphemes = self._analyze_word(word, pos, domain)
+                result.extend(morphemes)
+                pos += len(word)
+                continue
+            latin_match = re.match(r'[a-zA-Z]+', text[pos:])
+            if latin_match:
+                word = latin_match.group()
+                result.append(Morpheme(surface=word, lemma=word, pos='FOREIGN', start=pos, end=pos + len(word)))
+                pos += len(word)
+                continue
+            num_match = self.NUMBER_PATTERN.match(text[pos:])
+            if num_match:
+                num = num_match.group()
+                result.append(Morpheme(surface=num, lemma=num, pos='NUM', start=pos, end=pos + len(num)))
+                pos += len(num)
+                continue
+            result.append(Morpheme(surface=text[pos], lemma=text[pos], pos='PUNCT', start=pos, end=pos + 1))
+            pos += 1
+        return result
+
+    def _analyze_word(self, word: str, offset: int, domain: Domain) -> List[Morpheme]:
+        if word in self._user_dictionary:
+            lemma, pos_tag, _ = self._user_dictionary[word]
+            return [Morpheme(surface=word, lemma=lemma, pos=pos_tag, start=offset, end=offset + len(word))]
+        if word in self.function_words:
+            return [Morpheme(surface=word, lemma=word, pos=self.function_words[word], start=offset, end=offset + len(word))]
+        # Prefix separation
+        morphemes = []
+        remaining = word
+        curr_offset = offset
+        for prefix, pos_tag in self.prefixes.items():
+            if remaining.startswith(prefix) and len(remaining) > 1:
+                morphemes.append(Morpheme(surface=prefix, lemma=prefix, pos=pos_tag, start=curr_offset, end=curr_offset + 1))
+                curr_offset += 1
+                remaining = remaining[1:]
+                break
+        if remaining:
+            morphemes.append(Morpheme(surface=remaining, lemma=remaining, pos='N', start=curr_offset, end=offset + len(word)))
+        return morphemes if morphemes else [Morpheme(surface=word, lemma=word, pos='N', start=offset, end=offset + len(word))]
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        return []
+
+
+class GreekScriptAnalyzer(AdvancedMorphologicalAnalyzer):
+    """그리스 문자 기반 언어 템플릿 (Greek)"""
+
+    LANG_CODE = "el"
+    LANG_NAME = "Greek"
+
+    WORD_PATTERN = re.compile(r'[α-ωΑ-Ωάέήίόύώϊϋΐΰἀ-ῼ]+')
+    NUMBER_PATTERN = re.compile(r'[0-9]+')
+
+    def __init__(self):
+        super().__init__()
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'ο': 'DET', 'η': 'DET', 'το': 'DET', 'οι': 'DET', 'τα': 'DET',
+            'ένα': 'DET', 'μια': 'DET',
+            'εγώ': 'PRON', 'εσύ': 'PRON', 'αυτός': 'PRON', 'αυτή': 'PRON', 'αυτό': 'PRON',
+            'εμείς': 'PRON', 'εσείς': 'PRON', 'αυτοί': 'PRON', 'αυτές': 'PRON',
+            'και': 'CONJ', 'ή': 'CONJ', 'αλλά': 'CONJ', 'όμως': 'CONJ',
+            'σε': 'PREP', 'από': 'PREP', 'με': 'PREP', 'για': 'PREP',
+            'δεν': 'NEG', 'μην': 'NEG',
+        }
+
+    def _build_domain_dictionaries(self):
+        pass
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        if not text.strip():
+            return [AnalysisResult([])]
+        morphemes = self._analyze_text(text, domain)
+        result = AnalysisResult(morphemes=morphemes, score=1.0, domain=domain)
+        return [result]
+
+    def _analyze_text(self, text: str, domain: Domain) -> List[Morpheme]:
+        result = []
+        pos = 0
+        while pos < len(text):
+            if text[pos].isspace():
+                pos += 1
+                continue
+            word_match = self.WORD_PATTERN.match(text[pos:])
+            if word_match:
+                word = word_match.group()
+                morpheme = self._analyze_word(word, pos, domain)
+                result.append(morpheme)
+                pos += len(word)
+                continue
+            latin_match = re.match(r'[a-zA-Z]+', text[pos:])
+            if latin_match:
+                word = latin_match.group()
+                result.append(Morpheme(surface=word, lemma=word, pos='FOREIGN', start=pos, end=pos + len(word)))
+                pos += len(word)
+                continue
+            num_match = self.NUMBER_PATTERN.match(text[pos:])
+            if num_match:
+                num = num_match.group()
+                result.append(Morpheme(surface=num, lemma=num, pos='NUM', start=pos, end=pos + len(num)))
+                pos += len(num)
+                continue
+            result.append(Morpheme(surface=text[pos], lemma=text[pos], pos='PUNCT', start=pos, end=pos + 1))
+            pos += 1
+        return result
+
+    def _analyze_word(self, word: str, offset: int, domain: Domain) -> Morpheme:
+        word_lower = word.lower()
+        if word_lower in self._user_dictionary:
+            lemma, pos_tag, _ = self._user_dictionary[word_lower]
+            return Morpheme(surface=word, lemma=lemma, pos=pos_tag, start=offset, end=offset + len(word))
+        if word_lower in self.function_words:
+            return Morpheme(surface=word, lemma=word_lower, pos=self.function_words[word_lower], start=offset, end=offset + len(word))
+        # Morphological analysis
+        if word_lower.endswith(('ω', 'ει', 'ουν', 'ουμε')):  # Verb endings
+            return Morpheme(surface=word, lemma=word_lower, pos='V', start=offset, end=offset + len(word))
+        if word_lower.endswith(('ος', 'ης', 'ας', 'η', 'α', 'ο')):  # Noun endings
+            return Morpheme(surface=word, lemma=word_lower, pos='N', start=offset, end=offset + len(word))
+        if word[0].isupper():
+            return Morpheme(surface=word, lemma=word, pos='NP', start=offset, end=offset + len(word))
+        return Morpheme(surface=word, lemma=word_lower, pos='N', start=offset, end=offset + len(word))
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        return []
+
+
+class GeorgianScriptAnalyzer(AdvancedMorphologicalAnalyzer):
+    """조지아 문자 기반 언어 템플릿 (Georgian)"""
+
+    LANG_CODE = "ka"
+    LANG_NAME = "Georgian"
+
+    WORD_PATTERN = re.compile(r'[\u10A0-\u10FF\u2D00-\u2D2F]+')
+    NUMBER_PATTERN = re.compile(r'[0-9]+')
+
+    def __init__(self):
+        super().__init__()
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'მე': 'PRON', 'შენ': 'PRON', 'ის': 'PRON', 'ჩვენ': 'PRON', 'თქვენ': 'PRON', 'ისინი': 'PRON',
+            'და': 'CONJ', 'ან': 'CONJ', 'მაგრამ': 'CONJ',
+            'არ': 'NEG', 'არა': 'NEG',
+            '-ში': 'PSP', '-ზე': 'PSP', '-თან': 'PSP', '-დან': 'PSP',
+        }
+
+    def _build_domain_dictionaries(self):
+        pass
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        if not text.strip():
+            return [AnalysisResult([])]
+        morphemes = self._analyze_text(text, domain)
+        result = AnalysisResult(morphemes=morphemes, score=1.0, domain=domain)
+        return [result]
+
+    def _analyze_text(self, text: str, domain: Domain) -> List[Morpheme]:
+        result = []
+        pos = 0
+        while pos < len(text):
+            if text[pos].isspace():
+                pos += 1
+                continue
+            word_match = self.WORD_PATTERN.match(text[pos:])
+            if word_match:
+                word = word_match.group()
+                morpheme = self._analyze_word(word, pos, domain)
+                result.append(morpheme)
+                pos += len(word)
+                continue
+            num_match = self.NUMBER_PATTERN.match(text[pos:])
+            if num_match:
+                num = num_match.group()
+                result.append(Morpheme(surface=num, lemma=num, pos='NUM', start=pos, end=pos + len(num)))
+                pos += len(num)
+                continue
+            result.append(Morpheme(surface=text[pos], lemma=text[pos], pos='PUNCT', start=pos, end=pos + 1))
+            pos += 1
+        return result
+
+    def _analyze_word(self, word: str, offset: int, domain: Domain) -> Morpheme:
+        if word in self._user_dictionary:
+            lemma, pos_tag, _ = self._user_dictionary[word]
+            return Morpheme(surface=word, lemma=lemma, pos=pos_tag, start=offset, end=offset + len(word))
+        if word in self.function_words:
+            return Morpheme(surface=word, lemma=word, pos=self.function_words[word], start=offset, end=offset + len(word))
+        return Morpheme(surface=word, lemma=word, pos='N', start=offset, end=offset + len(word))
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        return []
+
+
+class ArmenianScriptAnalyzer(AdvancedMorphologicalAnalyzer):
+    """아르메니아 문자 기반 언어 템플릿 (Armenian)"""
+
+    LANG_CODE = "hy"
+    LANG_NAME = "Armenian"
+
+    WORD_PATTERN = re.compile(r'[\u0530-\u058F\uFB00-\uFB17]+')
+    NUMBER_PATTERN = re.compile(r'[0-9]+')
+
+    def __init__(self):
+        super().__init__()
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'ես': 'PRON', 'դdelays': 'PRON', 'նdelays': 'PRON',
+            'delays': 'CONJ', 'կdelays': 'CONJ',
+            'delays': 'NEG',
+        }
+
+    def _build_domain_dictionaries(self):
+        pass
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        if not text.strip():
+            return [AnalysisResult([])]
+        morphemes = self._analyze_text(text, domain)
+        result = AnalysisResult(morphemes=morphemes, score=1.0, domain=domain)
+        return [result]
+
+    def _analyze_text(self, text: str, domain: Domain) -> List[Morpheme]:
+        result = []
+        pos = 0
+        while pos < len(text):
+            if text[pos].isspace():
+                pos += 1
+                continue
+            word_match = self.WORD_PATTERN.match(text[pos:])
+            if word_match:
+                word = word_match.group()
+                morpheme = self._analyze_word(word, pos, domain)
+                result.append(morpheme)
+                pos += len(word)
+                continue
+            num_match = self.NUMBER_PATTERN.match(text[pos:])
+            if num_match:
+                num = num_match.group()
+                result.append(Morpheme(surface=num, lemma=num, pos='NUM', start=pos, end=pos + len(num)))
+                pos += len(num)
+                continue
+            result.append(Morpheme(surface=text[pos], lemma=text[pos], pos='PUNCT', start=pos, end=pos + 1))
+            pos += 1
+        return result
+
+    def _analyze_word(self, word: str, offset: int, domain: Domain) -> Morpheme:
+        if word in self._user_dictionary:
+            lemma, pos_tag, _ = self._user_dictionary[word]
+            return Morpheme(surface=word, lemma=lemma, pos=pos_tag, start=offset, end=offset + len(word))
+        return Morpheme(surface=word, lemma=word, pos='N', start=offset, end=offset + len(word))
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        return []
+
+
+class ThaiScriptAnalyzer(AdvancedMorphologicalAnalyzer):
+    """태국 문자 기반 언어 템플릿 (Thai) - 공백 없는 언어"""
+
+    LANG_CODE = "th"
+    LANG_NAME = "Thai"
+
+    WORD_PATTERN = re.compile(r'[\u0E00-\u0E7F]+')
+    NUMBER_PATTERN = re.compile(r'[0-9๐-๙]+')
+
+    def __init__(self):
+        super().__init__()
+
+    def _build_base_dictionary(self):
+        # Thai function words
+        self.function_words = {
+            'ฉัน': 'PRON', 'คุณ': 'PRON', 'เขา': 'PRON', 'เรา': 'PRON', 'พวกเขา': 'PRON',
+            'ที่': 'REL', 'ของ': 'PREP', 'ใน': 'PREP', 'บน': 'PREP', 'กับ': 'PREP',
+            'และ': 'CONJ', 'หรือ': 'CONJ', 'แต่': 'CONJ',
+            'ไม่': 'NEG', 'ไหม': 'Q',
+            'มาก': 'ADV', 'น้อย': 'ADV', 'ดี': 'ADJ',
+        }
+        # Common Thai words for word segmentation
+        self.common_words = set(self.function_words.keys())
+
+    def _build_domain_dictionaries(self):
+        pass
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        if not text.strip():
+            return [AnalysisResult([])]
+        morphemes = self._analyze_text(text, domain)
+        result = AnalysisResult(morphemes=morphemes, score=1.0, domain=domain)
+        return [result]
+
+    def _analyze_text(self, text: str, domain: Domain) -> List[Morpheme]:
+        """Thai text analysis - simplified word segmentation"""
+        result = []
+        pos = 0
+        while pos < len(text):
+            if text[pos].isspace():
+                pos += 1
+                continue
+            # Thai script
+            thai_match = self.WORD_PATTERN.match(text[pos:])
+            if thai_match:
+                chunk = thai_match.group()
+                # Simple maximum matching for word segmentation
+                words = self._segment_thai(chunk)
+                for word, wpos in words:
+                    if word in self.function_words:
+                        result.append(Morpheme(surface=word, lemma=word, pos=self.function_words[word], start=pos + wpos, end=pos + wpos + len(word)))
+                    else:
+                        result.append(Morpheme(surface=word, lemma=word, pos='N', start=pos + wpos, end=pos + wpos + len(word)))
+                pos += len(chunk)
+                continue
+            num_match = self.NUMBER_PATTERN.match(text[pos:])
+            if num_match:
+                num = num_match.group()
+                result.append(Morpheme(surface=num, lemma=num, pos='NUM', start=pos, end=pos + len(num)))
+                pos += len(num)
+                continue
+            result.append(Morpheme(surface=text[pos], lemma=text[pos], pos='PUNCT', start=pos, end=pos + 1))
+            pos += 1
+        return result
+
+    def _segment_thai(self, text: str) -> List[Tuple[str, int]]:
+        """Simple Thai word segmentation using maximum matching"""
+        words = []
+        pos = 0
+        while pos < len(text):
+            # Try to match known words (longest first)
+            matched = False
+            for length in range(min(10, len(text) - pos), 0, -1):
+                substr = text[pos:pos + length]
+                if substr in self.common_words or substr in self._user_dictionary:
+                    words.append((substr, pos))
+                    pos += length
+                    matched = True
+                    break
+            if not matched:
+                # Single character
+                words.append((text[pos], pos))
+                pos += 1
+        return words
+
+    def _analyze_word(self, word: str, offset: int, domain: Domain) -> Morpheme:
+        if word in self._user_dictionary:
+            lemma, pos_tag, _ = self._user_dictionary[word]
+            return Morpheme(surface=word, lemma=lemma, pos=pos_tag, start=offset, end=offset + len(word))
+        if word in self.function_words:
+            return Morpheme(surface=word, lemma=word, pos=self.function_words[word], start=offset, end=offset + len(word))
+        return Morpheme(surface=word, lemma=word, pos='N', start=offset, end=offset + len(word))
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        return []
+
+
+class EthiopicScriptAnalyzer(AdvancedMorphologicalAnalyzer):
+    """에티오피아 문자 기반 언어 템플릿 (Amharic, Tigrinya, etc.)"""
+
+    LANG_CODE = "am"
+    LANG_NAME = "Amharic"
+
+    WORD_PATTERN = re.compile(r'[\u1200-\u137F\u1380-\u139F\u2D80-\u2DDF]+')
+    NUMBER_PATTERN = re.compile(r'[0-9፩-፼]+')
+
+    def __init__(self):
+        super().__init__()
+
+    def _build_base_dictionary(self):
+        self.function_words = {
+            'እኔ': 'PRON', 'አንተ': 'PRON', 'እሱ': 'PRON', 'እሷ': 'PRON',
+            'እኛ': 'PRON', 'እናንተ': 'PRON', 'እነሱ': 'PRON',
+            'እና': 'CONJ', 'ወይም': 'CONJ', 'ግን': 'CONJ',
+            'አይ': 'NEG', 'የለም': 'NEG',
+            'ውስጥ': 'PREP', 'ላይ': 'PREP', 'ከ': 'PREP', 'ወደ': 'PREP',
+        }
+
+    def _build_domain_dictionaries(self):
+        pass
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        if not text.strip():
+            return [AnalysisResult([])]
+        morphemes = self._analyze_text(text, domain)
+        result = AnalysisResult(morphemes=morphemes, score=1.0, domain=domain)
+        return [result]
+
+    def _analyze_text(self, text: str, domain: Domain) -> List[Morpheme]:
+        result = []
+        pos = 0
+        while pos < len(text):
+            if text[pos].isspace():
+                pos += 1
+                continue
+            word_match = self.WORD_PATTERN.match(text[pos:])
+            if word_match:
+                word = word_match.group()
+                morpheme = self._analyze_word(word, pos, domain)
+                result.append(morpheme)
+                pos += len(word)
+                continue
+            num_match = self.NUMBER_PATTERN.match(text[pos:])
+            if num_match:
+                num = num_match.group()
+                result.append(Morpheme(surface=num, lemma=num, pos='NUM', start=pos, end=pos + len(num)))
+                pos += len(num)
+                continue
+            result.append(Morpheme(surface=text[pos], lemma=text[pos], pos='PUNCT', start=pos, end=pos + 1))
+            pos += 1
+        return result
+
+    def _analyze_word(self, word: str, offset: int, domain: Domain) -> Morpheme:
+        if word in self._user_dictionary:
+            lemma, pos_tag, _ = self._user_dictionary[word]
+            return Morpheme(surface=word, lemma=lemma, pos=pos_tag, start=offset, end=offset + len(word))
+        if word in self.function_words:
+            return Morpheme(surface=word, lemma=word, pos=self.function_words[word], start=offset, end=offset + len(word))
+        return Morpheme(surface=word, lemma=word, pos='N', start=offset, end=offset + len(word))
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        return []