tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/arabic_advanced.py

@@ -0,0 +1,247 @@
+"""
+Arabic Advanced Morphological Analyzer
+======================================
+
+5가지 고급 기능을 지원하는 아랍어 형태소 분석기
+
+특징:
+- RTL (Right-to-Left) 문자 처리
+- 어근-어형 시스템 (Root-Pattern)
+- 접두사/접미사 분리
+"""
+
+import re
+from typing import List, Tuple, Dict, Optional
+
+from .advanced_base import (
+    AdvancedMorphologicalAnalyzer, Morpheme, AnalysisResult, NBestResult, Domain
+)
+
+
+class ArabicAdvancedAnalyzer(AdvancedMorphologicalAnalyzer):
+    """아랍어 고급 형태소 분석기"""
+
+    LANG_CODE = "ar"
+    LANG_NAME = "Arabic"
+
+    # 아랍어 문자 패턴
+    ARABIC_PATTERN = re.compile(r'[\u0600-\u06FF\u0750-\u077F]+')
+    NUMBER_PATTERN = re.compile(r'[0-9٠-٩]+(?:[.,][0-9٠-٩]+)?')
+
+    def __init__(self):
+        super().__init__()
+
+    def _build_base_dictionary(self):
+        """기본 사전 구축"""
+
+        # 접두사 (Prefixes)
+        self.prefixes = {
+            'ال': 'DEF',  # 정관사 (the)
+            'و': 'CONJ',  # 그리고 (and)
+            'ف': 'CONJ',  # 그러면 (so)
+            'ب': 'PREP',  # ~로 (with/by)
+            'ك': 'PREP',  # ~처럼 (like)
+            'ل': 'PREP',  # ~에게 (to/for)
+            'س': 'FUT',   # 미래 표지
+            'سوف': 'FUT', # 미래 표지
+        }
+
+        # 접미사 (대명사 접미사)
+        self.suffixes = {
+            'ي': 'PRON',   # 나의
+            'ك': 'PRON',   # 너의
+            'ه': 'PRON',   # 그의
+            'ها': 'PRON',  # 그녀의
+            'نا': 'PRON',  # 우리의
+            'كم': 'PRON',  # 너희의
+            'هم': 'PRON',  # 그들의
+            'ة': 'FEM',    # 여성 어미
+            'ات': 'PL',    # 복수 어미 (여성)
+            'ون': 'PL',    # 복수 어미 (남성)
+            'ين': 'PL',    # 복수 어미 (남성)
+        }
+
+        # 대명사
+        self.pronouns = {
+            'أنا': 'PRON', 'أنت': 'PRON', 'أنتِ': 'PRON',
+            'هو': 'PRON', 'هي': 'PRON',
+            'نحن': 'PRON', 'أنتم': 'PRON', 'هم': 'PRON', 'هن': 'PRON',
+            'هذا': 'DEM', 'هذه': 'DEM', 'ذلك': 'DEM', 'تلك': 'DEM',
+            'من': 'REL', 'ما': 'REL', 'الذي': 'REL', 'التي': 'REL',
+        }
+
+        # 전치사
+        self.prepositions = {
+            'في': 'PREP', 'من': 'PREP', 'إلى': 'PREP', 'على': 'PREP',
+            'عن': 'PREP', 'مع': 'PREP', 'بين': 'PREP', 'حول': 'PREP',
+            'قبل': 'PREP', 'بعد': 'PREP', 'تحت': 'PREP', 'فوق': 'PREP',
+            'عند': 'PREP', 'خلال': 'PREP', 'منذ': 'PREP', 'دون': 'PREP',
+        }
+
+        # 접속사
+        self.conjunctions = {
+            'و': 'CONJ', 'أو': 'CONJ', 'لكن': 'CONJ', 'بل': 'CONJ',
+            'أن': 'CONJ', 'إن': 'CONJ', 'لأن': 'CONJ', 'إذا': 'CONJ',
+            'حتى': 'CONJ', 'كما': 'CONJ', 'عندما': 'CONJ', 'بينما': 'CONJ',
+        }
+
+        # 부사
+        self.adverbs = {
+            'جداً': 'ADV', 'كثيراً': 'ADV', 'قليلاً': 'ADV',
+            'دائماً': 'ADV', 'أبداً': 'ADV', 'أحياناً': 'ADV',
+            'الآن': 'ADV', 'غداً': 'ADV', 'أمس': 'ADV', 'اليوم': 'ADV',
+            'هنا': 'ADV', 'هناك': 'ADV', 'فقط': 'ADV', 'أيضاً': 'ADV',
+        }
+
+        # 부정어
+        self.negation = {
+            'لا': 'NEG', 'لم': 'NEG', 'لن': 'NEG', 'ما': 'NEG',
+            'ليس': 'NEG', 'ليست': 'NEG', 'ليسوا': 'NEG',
+        }
+
+        # 일반 명사 (고빈도)
+        self.common_nouns = {
+            'الله': 'NP',  # Allah
+            'رسول': 'NC',  # messenger
+            'كتاب': 'NC',  # book
+            'يوم': 'NC',   # day
+            'سنة': 'NC',   # year
+            'ناس': 'NC',   # people
+            'بيت': 'NC',   # house
+            'عمل': 'NC',   # work
+            'دولة': 'NC',  # state
+            'شركة': 'NC',  # company
+        }
+
+    def _build_domain_dictionaries(self):
+        """도메인별 사전"""
+        self._domain_dictionaries[Domain.TECH] = {
+            'تفاحة': ('Apple', 'NP'),  # Apple (company)
+            'سحابة': ('cloud', 'NC'),  # cloud computing
+        }
+        self._domain_dictionaries[Domain.FOOD] = {
+            'تفاحة': ('تفاحة', 'NC'),  # apple (fruit)
+        }
+        self._domain_dictionaries[Domain.FINANCE] = {
+            'بنك': ('بنك', 'NC'),  # bank
+            'سهم': ('سهم', 'NC'),  # stock
+        }
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        if not text or not text.strip():
+            return [AnalysisResult([])]
+        morphemes = self._analyze_text(text, domain)
+        result = AnalysisResult(morphemes=morphemes, score=1.0, domain=domain)
+        result.score = self._score_analysis(result)
+        return [result]
+
+    def _analyze_text(self, text: str, domain: Domain) -> List[Morpheme]:
+        result = []
+        pos = 0
+        while pos < len(text):
+            if text[pos].isspace():
+                pos += 1
+                continue
+
+            arabic_match = self.ARABIC_PATTERN.match(text[pos:])
+            if arabic_match:
+                word = arabic_match.group()
+                morphemes = self._analyze_word(word, pos, domain)
+                result.extend(morphemes)
+                pos += len(word)
+                continue
+
+            # 라틴 문자 (외래어)
+            latin_match = re.match(r'[a-zA-Z]+', text[pos:])
+            if latin_match:
+                word = latin_match.group()
+                result.append(Morpheme(surface=word, lemma=word, pos='FOREIGN', start=pos, end=pos + len(word)))
+                pos += len(word)
+                continue
+
+            num_match = self.NUMBER_PATTERN.match(text[pos:])
+            if num_match:
+                num = num_match.group()
+                result.append(Morpheme(surface=num, lemma=num, pos='NUM', start=pos, end=pos + len(num)))
+                pos += len(num)
+                continue
+
+            result.append(Morpheme(surface=text[pos], lemma=text[pos], pos='PUNCT', start=pos, end=pos + 1))
+            pos += 1
+        return result
+
+    def _analyze_word(self, word: str, offset: int, domain: Domain) -> List[Morpheme]:
+        """단어 분석 (접두사/접미사 분리)"""
+        morphemes = []
+
+        # 런타임 사전
+        if word in self._user_dictionary:
+            lemma, pos_tag, _ = self._user_dictionary[word]
+            return [Morpheme(surface=word, lemma=lemma, pos=pos_tag, start=offset, end=offset + len(word))]
+
+        # 도메인 사전
+        domain_sense = self._get_domain_sense(word, domain)
+        if domain_sense:
+            return [Morpheme(surface=word, lemma=domain_sense[0], pos=domain_sense[1], start=offset, end=offset + len(word))]
+
+        # 기능어
+        if word in self.pronouns:
+            return [Morpheme(surface=word, lemma=word, pos='PRON', start=offset, end=offset + len(word))]
+        if word in self.prepositions:
+            return [Morpheme(surface=word, lemma=word, pos='PREP', start=offset, end=offset + len(word))]
+        if word in self.conjunctions:
+            return [Morpheme(surface=word, lemma=word, pos='CONJ', start=offset, end=offset + len(word))]
+        if word in self.adverbs:
+            return [Morpheme(surface=word, lemma=word, pos='ADV', start=offset, end=offset + len(word))]
+        if word in self.negation:
+            return [Morpheme(surface=word, lemma=word, pos='NEG', start=offset, end=offset + len(word))]
+        if word in self.common_nouns:
+            return [Morpheme(surface=word, lemma=word, pos=self.common_nouns[word], start=offset, end=offset + len(word))]
+
+        # 접두사 분리
+        current_offset = offset
+        remaining = word
+
+        # 정관사 ال 분리
+        if remaining.startswith('ال') and len(remaining) > 2:
+            morphemes.append(Morpheme(surface='ال', lemma='ال', pos='DEF', start=current_offset, end=current_offset + 2))
+            current_offset += 2
+            remaining = remaining[2:]
+
+        # 접속사/전치사 접두사
+        for prefix, pos_tag in self.prefixes.items():
+            if remaining.startswith(prefix) and len(remaining) > len(prefix):
+                morphemes.append(Morpheme(surface=prefix, lemma=prefix, pos=pos_tag, start=current_offset, end=current_offset + len(prefix)))
+                current_offset += len(prefix)
+                remaining = remaining[len(prefix):]
+                break
+
+        # 남은 어간
+        if remaining:
+            stem_end = current_offset + len(remaining)
+
+            # 접미사 분리 시도
+            for suffix, pos_tag in sorted(self.suffixes.items(), key=lambda x: -len(x[0])):
+                if remaining.endswith(suffix) and len(remaining) > len(suffix):
+                    stem = remaining[:-len(suffix)]
+                    morphemes.append(Morpheme(surface=stem, lemma=stem, pos='NC', start=current_offset, end=current_offset + len(stem)))
+                    morphemes.append(Morpheme(surface=suffix, lemma=suffix, pos=pos_tag, start=current_offset + len(stem), end=stem_end))
+                    return morphemes
+
+            # 접미사 없음
+            morphemes.append(Morpheme(surface=remaining, lemma=remaining, pos='NC', start=current_offset, end=stem_end))
+
+        return morphemes if morphemes else [Morpheme(surface=word, lemma=word, pos='NC', start=offset, end=offset + len(word))]
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        alternatives = []
+        other_domains = [d for d in Domain if d != domain][:count]
+        for alt_domain in other_domains:
+            morphemes = self._analyze_text(text, alt_domain)
+            result = AnalysisResult(morphemes=morphemes, score=0.8, domain=alt_domain)
+            result.score = self._score_analysis(result) * 0.9
+            alternatives.append(result)
+        return alternatives
+
+
+ArabicAnalyzer = ArabicAdvancedAnalyzer