tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/universal_fallback.py

@@ -0,0 +1,398 @@
+"""
+Universal Fallback Morphological Analyzer
+=========================================
+
+모든 언어를 커버하는 범용 형태소 분석기
+언어별 분석기가 없는 경우 자동으로 사용됨
+
+Features:
+- Unicode script detection
+- Automatic word boundary detection
+- RTL/LTR handling
+- Any language support
+"""
+
+import re
+import unicodedata
+from typing import List, Dict, Optional, Set, Tuple
+
+from .advanced_base import (
+    AdvancedMorphologicalAnalyzer, Morpheme, AnalysisResult, Domain
+)
+
+
+class UniversalFallbackAnalyzer(AdvancedMorphologicalAnalyzer):
+    """
+    Universal fallback analyzer for any language
+
+    Handles:
+    - All Unicode scripts automatically
+    - Mixed script texts
+    - Unknown languages
+    - Automatic script detection
+    """
+
+    LANG_CODE = "xx"  # Universal code
+    LANG_NAME = "Universal"
+
+    # Unicode script categories (simplified)
+    SCRIPT_PATTERNS = {
+        'latin': re.compile(r'[\u0041-\u007A\u00C0-\u024F\u1E00-\u1EFF]+'),
+        'cyrillic': re.compile(r'[\u0400-\u04FF\u0500-\u052F]+'),
+        'greek': re.compile(r'[\u0370-\u03FF\u1F00-\u1FFF]+'),
+        'arabic': re.compile(r'[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF]+'),
+        'hebrew': re.compile(r'[\u0590-\u05FF]+'),
+        'devanagari': re.compile(r'[\u0900-\u097F]+'),
+        'bengali': re.compile(r'[\u0980-\u09FF]+'),
+        'tamil': re.compile(r'[\u0B80-\u0BFF]+'),
+        'telugu': re.compile(r'[\u0C00-\u0C7F]+'),
+        'kannada': re.compile(r'[\u0C80-\u0CFF]+'),
+        'malayalam': re.compile(r'[\u0D00-\u0D7F]+'),
+        'thai': re.compile(r'[\u0E00-\u0E7F]+'),
+        'lao': re.compile(r'[\u0E80-\u0EFF]+'),
+        'myanmar': re.compile(r'[\u1000-\u109F]+'),
+        'khmer': re.compile(r'[\u1780-\u17FF]+'),
+        'tibetan': re.compile(r'[\u0F00-\u0FFF]+'),
+        'georgian': re.compile(r'[\u10A0-\u10FF\u2D00-\u2D2F]+'),
+        'armenian': re.compile(r'[\u0530-\u058F]+'),
+        'hangul': re.compile(r'[\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F]+'),
+        'hiragana': re.compile(r'[\u3040-\u309F]+'),
+        'katakana': re.compile(r'[\u30A0-\u30FF]+'),
+        'cjk': re.compile(r'[\u4E00-\u9FFF\u3400-\u4DBF]+'),
+        'ethiopic': re.compile(r'[\u1200-\u137F]+'),
+        'sinhala': re.compile(r'[\u0D80-\u0DFF]+'),
+        'gujarati': re.compile(r'[\u0A80-\u0AFF]+'),
+        'gurmukhi': re.compile(r'[\u0A00-\u0A7F]+'),
+        'oriya': re.compile(r'[\u0B00-\u0B7F]+'),
+    }
+
+    # Combined word pattern for any script
+    WORD_PATTERN = re.compile(
+        r'[\u0041-\u007A\u00C0-\u024F\u1E00-\u1EFF'  # Latin
+        r'\u0400-\u052F'  # Cyrillic
+        r'\u0370-\u03FF\u1F00-\u1FFF'  # Greek
+        r'\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF'  # Arabic
+        r'\u0590-\u05FF'  # Hebrew
+        r'\u0900-\u097F'  # Devanagari
+        r'\u0980-\u09FF'  # Bengali
+        r'\u0A00-\u0A7F'  # Gurmukhi
+        r'\u0A80-\u0AFF'  # Gujarati
+        r'\u0B00-\u0B7F'  # Oriya
+        r'\u0B80-\u0BFF'  # Tamil
+        r'\u0C00-\u0C7F'  # Telugu
+        r'\u0C80-\u0CFF'  # Kannada
+        r'\u0D00-\u0DFF'  # Malayalam/Sinhala
+        r'\u0E00-\u0E7F'  # Thai
+        r'\u0E80-\u0EFF'  # Lao
+        r'\u0F00-\u0FFF'  # Tibetan
+        r'\u1000-\u109F'  # Myanmar
+        r'\u1200-\u137F'  # Ethiopic
+        r'\u1780-\u17FF'  # Khmer
+        r'\u10A0-\u10FF\u2D00-\u2D2F'  # Georgian
+        r'\u0530-\u058F'  # Armenian
+        r'\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F'  # Hangul
+        r'\u3040-\u309F'  # Hiragana
+        r'\u30A0-\u30FF'  # Katakana
+        r'\u4E00-\u9FFF\u3400-\u4DBF'  # CJK
+        r']+',
+        re.UNICODE
+    )
+
+    # Number patterns (various scripts)
+    NUMBER_PATTERN = re.compile(
+        r'[0-9'
+        r'\u0660-\u0669'  # Arabic-Indic
+        r'\u06F0-\u06F9'  # Extended Arabic-Indic
+        r'\u0966-\u096F'  # Devanagari
+        r'\u09E6-\u09EF'  # Bengali
+        r'\u0A66-\u0A6F'  # Gurmukhi
+        r'\u0AE6-\u0AEF'  # Gujarati
+        r'\u0B66-\u0B6F'  # Oriya
+        r'\u0BE6-\u0BEF'  # Tamil
+        r'\u0C66-\u0C6F'  # Telugu
+        r'\u0CE6-\u0CEF'  # Kannada
+        r'\u0D66-\u0D6F'  # Malayalam
+        r'\u0E50-\u0E59'  # Thai
+        r'\u0ED0-\u0ED9'  # Lao
+        r'\u0F20-\u0F29'  # Tibetan
+        r'\u1040-\u1049'  # Myanmar
+        r'\u17E0-\u17E9'  # Khmer
+        r']+'
+    )
+
+    def __init__(self):
+        super().__init__()
+        self._detected_script: Optional[str] = None
+
+    def _build_base_dictionary(self):
+        """Universal basic dictionary"""
+        self.function_words: Dict[str, str] = {}
+
+    def _build_domain_dictionaries(self):
+        """No domain dictionaries for universal"""
+        pass
+
+    def detect_script(self, text: str) -> str:
+        """Detect primary script of text"""
+        script_counts: Dict[str, int] = {}
+
+        for script_name, pattern in self.SCRIPT_PATTERNS.items():
+            matches = pattern.findall(text)
+            if matches:
+                script_counts[script_name] = sum(len(m) for m in matches)
+
+        if not script_counts:
+            return 'unknown'
+
+        return max(script_counts, key=script_counts.get)
+
+    def detect_language_hint(self, text: str) -> str:
+        """Try to detect language from script"""
+        script = self.detect_script(text)
+
+        # Script to likely language mapping
+        script_lang_map = {
+            'hangul': 'ko',
+            'hiragana': 'ja',
+            'katakana': 'ja',
+            'cjk': 'zh',  # Could be zh/ja/ko
+            'devanagari': 'hi',
+            'bengali': 'bn',
+            'tamil': 'ta',
+            'telugu': 'te',
+            'kannada': 'kn',
+            'malayalam': 'ml',
+            'gujarati': 'gu',
+            'gurmukhi': 'pa',
+            'oriya': 'or',
+            'thai': 'th',
+            'lao': 'lo',
+            'myanmar': 'my',
+            'khmer': 'km',
+            'tibetan': 'bo',
+            'georgian': 'ka',
+            'armenian': 'hy',
+            'ethiopic': 'am',
+            'sinhala': 'si',
+            'hebrew': 'he',
+            'arabic': 'ar',
+            'greek': 'el',
+            'cyrillic': 'ru',
+            'latin': 'en',
+        }
+
+        return script_lang_map.get(script, 'xx')
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        if not text or not text.strip():
+            return [AnalysisResult([])]
+
+        self._detected_script = self.detect_script(text)
+        morphemes = self._analyze_text(text, domain)
+        result = AnalysisResult(morphemes=morphemes, score=1.0, domain=domain)
+        result.score = self._score_analysis(result)
+        return [result]
+
+    def _analyze_text(self, text: str, domain: Domain) -> List[Morpheme]:
+        """Universal tokenization using Unicode properties"""
+        result = []
+        pos = 0
+
+        while pos < len(text):
+            # Skip whitespace
+            if text[pos].isspace():
+                pos += 1
+                continue
+
+            # Try number first
+            num_match = self.NUMBER_PATTERN.match(text[pos:])
+            if num_match:
+                num = num_match.group()
+                result.append(Morpheme(
+                    surface=num, lemma=num, pos='NUM',
+                    start=pos, end=pos + len(num)
+                ))
+                pos += len(num)
+                continue
+
+            # Try word pattern
+            word_match = self.WORD_PATTERN.match(text[pos:])
+            if word_match:
+                word = word_match.group()
+                morpheme = self._analyze_word(word, pos, domain)
+                result.append(morpheme)
+                pos += len(word)
+                continue
+
+            # Check for CJK characters (needs character-by-character for Chinese)
+            if self._is_cjk(text[pos]):
+                # Group consecutive CJK
+                end = pos
+                while end < len(text) and self._is_cjk(text[end]):
+                    end += 1
+                word = text[pos:end]
+                result.append(Morpheme(
+                    surface=word, lemma=word, pos='N',
+                    start=pos, end=end
+                ))
+                pos = end
+                continue
+
+            # Punctuation or unknown
+            char = text[pos]
+            if unicodedata.category(char).startswith('P'):
+                result.append(Morpheme(
+                    surface=char, lemma=char, pos='PUNCT',
+                    start=pos, end=pos + 1
+                ))
+            else:
+                result.append(Morpheme(
+                    surface=char, lemma=char, pos='X',
+                    start=pos, end=pos + 1
+                ))
+            pos += 1
+
+        return result
+
+    def _is_cjk(self, char: str) -> bool:
+        """Check if character is CJK"""
+        if not char:
+            return False
+        code = ord(char)
+        return (
+            0x4E00 <= code <= 0x9FFF or  # CJK Unified
+            0x3400 <= code <= 0x4DBF or  # CJK Extension A
+            0x20000 <= code <= 0x2A6DF or  # CJK Extension B
+            0x2A700 <= code <= 0x2B73F or  # CJK Extension C
+            0x2B740 <= code <= 0x2B81F or  # CJK Extension D
+            0xF900 <= code <= 0xFAFF or  # CJK Compatibility
+            0x2F00 <= code <= 0x2FDF  # Kangxi Radicals
+        )
+
+    def _analyze_word(self, word: str, offset: int, domain: Domain) -> Morpheme:
+        """Analyze single word"""
+
+        # Check user dictionary
+        if word in self._user_dictionary:
+            lemma, pos_tag, _ = self._user_dictionary[word]
+            return Morpheme(
+                surface=word, lemma=lemma, pos=pos_tag,
+                start=offset, end=offset + len(word)
+            )
+
+        # Check domain dictionary
+        domain_sense = self._get_domain_sense(word, domain)
+        if domain_sense:
+            return Morpheme(
+                surface=word, lemma=domain_sense[0], pos=domain_sense[1],
+                start=offset, end=offset + len(word)
+            )
+
+        # Heuristic POS tagging based on Unicode category
+        pos = self._guess_pos(word)
+
+        return Morpheme(
+            surface=word, lemma=word.lower() if word.isascii() else word,
+            pos=pos, start=offset, end=offset + len(word)
+        )
+
+    def _guess_pos(self, word: str) -> str:
+        """Heuristic POS guessing based on character properties"""
+        if not word:
+            return 'X'
+
+        # All uppercase -> proper noun or acronym
+        if word.isupper() and len(word) > 1:
+            return 'NNP'
+
+        # Title case -> proper noun
+        if word.istitle():
+            return 'NNP'
+
+        # Contains digits -> number-like
+        if any(c.isdigit() for c in word):
+            return 'NUM'
+
+        # Default to noun
+        return 'N'
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        """Generate alternative analyses"""
+        alternatives = []
+        other_domains = [d for d in Domain if d != domain][:count]
+
+        for alt_domain in other_domains:
+            morphemes = self._analyze_text(text, alt_domain)
+            result = AnalysisResult(morphemes=morphemes, score=0.8, domain=alt_domain)
+            result.score = self._score_analysis(result) * 0.9
+            alternatives.append(result)
+
+        return alternatives
+
+
+# Language registry with fallback
+class AnalyzerRegistry:
+    """
+    Global registry for morphological analyzers
+    Auto-selects appropriate analyzer based on language code or script detection
+    """
+
+    _analyzers: Dict[str, type] = {}
+    _instances: Dict[str, AdvancedMorphologicalAnalyzer] = {}
+    _fallback = UniversalFallbackAnalyzer
+
+    @classmethod
+    def register(cls, lang_code: str, analyzer_class: type):
+        """Register an analyzer for a language code"""
+        cls._analyzers[lang_code.lower()] = analyzer_class
+
+    @classmethod
+    def get(cls, lang_code: str) -> AdvancedMorphologicalAnalyzer:
+        """Get analyzer instance for language code"""
+        lang_code = lang_code.lower()
+
+        if lang_code not in cls._instances:
+            if lang_code in cls._analyzers:
+                cls._instances[lang_code] = cls._analyzers[lang_code]()
+            else:
+                # Use fallback
+                cls._instances[lang_code] = cls._fallback()
+
+        return cls._instances[lang_code]
+
+    @classmethod
+    def get_for_text(cls, text: str) -> AdvancedMorphologicalAnalyzer:
+        """Auto-detect language and get appropriate analyzer"""
+        fallback = cls._fallback()
+        detected_lang = fallback.detect_language_hint(text)
+        return cls.get(detected_lang)
+
+    @classmethod
+    def supported_languages(cls) -> List[str]:
+        """List all registered language codes"""
+        return list(cls._analyzers.keys())
+
+    @classmethod
+    def clear_cache(cls):
+        """Clear instance cache"""
+        cls._instances.clear()
+
+
+# Convenience function
+def get_analyzer(lang_code: str = None, text: str = None) -> AdvancedMorphologicalAnalyzer:
+    """
+    Get morphological analyzer
+
+    Args:
+        lang_code: Language code (e.g., 'ko', 'en', 'ja')
+        text: Text to analyze (for auto-detection)
+
+    Returns:
+        Appropriate analyzer instance
+    """
+    if lang_code:
+        return AnalyzerRegistry.get(lang_code)
+    elif text:
+        return AnalyzerRegistry.get_for_text(text)
+    else:
+        return UniversalFallbackAnalyzer()