tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/__init__.py

@@ -0,0 +1,395 @@
+"""
+TokMor Morphology - 통합 형태소 분석기
+======================================
+
+350+ 언어 지원 형태소 분석기
+- 특화 분석기 (74개 언어): 규칙 기반 고급 분석
+- 형태소 사전/규칙 모델 (333개 언어): 사전 기반 lemmatization (온라인 호출 없음)
+
+간편 사용:
+    from tokmor.morphology import lemmatize, analyze
+
+    # Lemmatization
+    lemmatize('running', 'en')  # 'run'
+    lemmatize('했다', 'ko')     # '하다'
+
+    # 상세 분석
+    results = analyze('The cats are running', 'en')
+    for r in results:
+        print(f'{r.word} -> {r.lemma}')
+
+지원 언어:
+- Tier 1 (10): ko, ja, zh, en, de, fr, es, ru, ar, hi
+- Tier 2 (15): pt, it, nl, pl, tr, vi, th, id, he, fa, uk, el, cs, ro, sv
+- Tier 3 (21): no, da, fi, hu, bg, hr, sr, sk, sl, lt, lv, et, ca, eu, gl, cy, ga, is, mt, sq, mk
+- Tier 4 (26+): bn, ta, te, mr, gu, kn, ml, pa, ur, ne, si, my, km, lo, tl, sw, am, yo, ha, zu, af, ka, hy, az, kk, uz, mn
+- 형태소 사전/규칙 모델 (333): 추가 언어 지원
+"""
+
+# Base classes
+from .advanced_base import (
+    AdvancedMorphologicalAnalyzer,
+    Morpheme,
+    AnalysisResult,
+    NBestResult,
+    Domain
+)
+
+# Legacy analyzers (backward compatibility)
+from .korean import KoreanAnalyzer as KoreanAnalyzerLegacy
+from .japanese import JapaneseAnalyzer as JapaneseAnalyzerLegacy
+from .chinese import ChineseAnalyzer as ChineseAnalyzerLegacy
+
+# =============================================================================
+# Tier 1 - Advanced analyzers (10 languages)
+# =============================================================================
+from .korean_advanced import KoreanAdvancedAnalyzer, KoreanAnalyzer
+from .japanese_advanced import JapaneseAdvancedAnalyzer, JapaneseAnalyzer
+from .chinese_advanced import ChineseAdvancedAnalyzer, ChineseAnalyzer
+from .english_advanced import EnglishAdvancedAnalyzer, EnglishAnalyzer
+from .german_advanced import GermanAdvancedAnalyzer, GermanAnalyzer
+from .french_advanced import FrenchAdvancedAnalyzer, FrenchAnalyzer
+from .spanish_advanced import SpanishAdvancedAnalyzer, SpanishAnalyzer
+from .russian_advanced import RussianAdvancedAnalyzer, RussianAnalyzer
+from .arabic_advanced import ArabicAdvancedAnalyzer, ArabicAnalyzer
+from .hindi_advanced import HindiAdvancedAnalyzer, HindiAnalyzer
+
+# =============================================================================
+# Tier 2 - Major regional languages (15 languages)
+# =============================================================================
+from .tier2 import (
+    PortugueseAnalyzer,
+    ItalianAnalyzer,
+    DutchAnalyzer,
+    PolishAnalyzer,
+    TurkishAnalyzer,
+    VietnameseAnalyzer,
+    ThaiAnalyzer,
+    IndonesianAnalyzer,
+    HebrewAnalyzer,
+    PersianAnalyzer,
+    UkrainianAnalyzer,
+    GreekAnalyzer,
+    CzechAnalyzer,
+    RomanianAnalyzer,
+    SwedishAnalyzer,
+)
+
+# =============================================================================
+# Tier 3 - Regional languages (21 languages)
+# =============================================================================
+from .tier3 import (
+    NorwegianAnalyzer,
+    DanishAnalyzer,
+    FinnishAnalyzer,
+    HungarianAnalyzer,
+    BulgarianAnalyzer,
+    CroatianAnalyzer,
+    SerbianAnalyzer,
+    SlovakAnalyzer,
+    SlovenianAnalyzer,
+    LithuanianAnalyzer,
+    LatvianAnalyzer,
+    EstonianAnalyzer,
+    CatalanAnalyzer,
+    BasqueAnalyzer,
+    GalicianAnalyzer,
+    WelshAnalyzer,
+    IrishAnalyzer,
+    IcelandicAnalyzer,
+    MalteseAnalyzer,
+    AlbanianAnalyzer,
+    MacedonianAnalyzer,
+)
+
+# =============================================================================
+# Tier 4 - Extended global coverage (26+ languages)
+# =============================================================================
+from .tier4 import (
+    # South Asian
+    BengaliAnalyzer,
+    TamilAnalyzer,
+    TeluguAnalyzer,
+    MarathiAnalyzer,
+    GujaratiAnalyzer,
+    KannadaAnalyzer,
+    MalayalamAnalyzer,
+    PunjabiAnalyzer,
+    UrduAnalyzer,
+    NepaliAnalyzer,
+    SinhalaAnalyzer,
+    # Southeast Asian
+    MyanmarAnalyzer,
+    KhmerAnalyzer,
+    LaoAnalyzer,
+    TagalogAnalyzer,
+    MalayAnalyzer,
+    # African
+    SwahiliAnalyzer,
+    AmharicAnalyzer,
+    YorubaAnalyzer,
+    HausaAnalyzer,
+    ZuluAnalyzer,
+    AfrikaansAnalyzer,
+    # Caucasian
+    GeorgianAnalyzer,
+    ArmenianAnalyzer,
+    AzerbaijaniAnalyzer,
+    # Central Asian
+    KazakhAnalyzer,
+    UzbekAnalyzer,
+    MongolianAnalyzer,
+)
+
+# =============================================================================
+# Universal Fallback
+# =============================================================================
+from .universal_fallback import (
+    UniversalFallbackAnalyzer,
+    AnalyzerRegistry,
+    get_analyzer as get_analyzer_auto,
+)
+
+# =============================================================================
+# Analyzer Registry - All 72+ languages
+# =============================================================================
+ANALYZERS = {
+    # Tier 1 (10)
+    'ko': KoreanAdvancedAnalyzer,
+    'ja': JapaneseAdvancedAnalyzer,
+    'zh': ChineseAdvancedAnalyzer,
+    'en': EnglishAdvancedAnalyzer,
+    'de': GermanAdvancedAnalyzer,
+    'fr': FrenchAdvancedAnalyzer,
+    'es': SpanishAdvancedAnalyzer,
+    'ru': RussianAdvancedAnalyzer,
+    'ar': ArabicAdvancedAnalyzer,
+    'hi': HindiAdvancedAnalyzer,
+
+    # Tier 2 (15)
+    'pt': PortugueseAnalyzer,
+    'it': ItalianAnalyzer,
+    'nl': DutchAnalyzer,
+    'pl': PolishAnalyzer,
+    'tr': TurkishAnalyzer,
+    'vi': VietnameseAnalyzer,
+    'th': ThaiAnalyzer,
+    'id': IndonesianAnalyzer,
+    'he': HebrewAnalyzer,
+    'fa': PersianAnalyzer,
+    'uk': UkrainianAnalyzer,
+    'el': GreekAnalyzer,
+    'cs': CzechAnalyzer,
+    'ro': RomanianAnalyzer,
+    'sv': SwedishAnalyzer,
+
+    # Tier 3 (21)
+    'no': NorwegianAnalyzer,
+    'da': DanishAnalyzer,
+    'fi': FinnishAnalyzer,
+    'hu': HungarianAnalyzer,
+    'bg': BulgarianAnalyzer,
+    'hr': CroatianAnalyzer,
+    'sr': SerbianAnalyzer,
+    'sk': SlovakAnalyzer,
+    'sl': SlovenianAnalyzer,
+    'lt': LithuanianAnalyzer,
+    'lv': LatvianAnalyzer,
+    'et': EstonianAnalyzer,
+    'ca': CatalanAnalyzer,
+    'eu': BasqueAnalyzer,
+    'gl': GalicianAnalyzer,
+    'cy': WelshAnalyzer,
+    'ga': IrishAnalyzer,
+    'is': IcelandicAnalyzer,
+    'mt': MalteseAnalyzer,
+    'sq': AlbanianAnalyzer,
+    'mk': MacedonianAnalyzer,
+
+    # Tier 4 - South Asian (11)
+    'bn': BengaliAnalyzer,
+    'ta': TamilAnalyzer,
+    'te': TeluguAnalyzer,
+    'mr': MarathiAnalyzer,
+    'gu': GujaratiAnalyzer,
+    'kn': KannadaAnalyzer,
+    'ml': MalayalamAnalyzer,
+    'pa': PunjabiAnalyzer,
+    'ur': UrduAnalyzer,
+    'ne': NepaliAnalyzer,
+    'si': SinhalaAnalyzer,
+
+    # Tier 4 - Southeast Asian (5)
+    'my': MyanmarAnalyzer,
+    'km': KhmerAnalyzer,
+    'lo': LaoAnalyzer,
+    'tl': TagalogAnalyzer,
+    'ms': MalayAnalyzer,
+
+    # Tier 4 - African (6)
+    'sw': SwahiliAnalyzer,
+    'am': AmharicAnalyzer,
+    'yo': YorubaAnalyzer,
+    'ha': HausaAnalyzer,
+    'zu': ZuluAnalyzer,
+    'af': AfrikaansAnalyzer,
+
+    # Tier 4 - Caucasian (3)
+    'ka': GeorgianAnalyzer,
+    'hy': ArmenianAnalyzer,
+    'az': AzerbaijaniAnalyzer,
+
+    # Tier 4 - Central Asian (3)
+    'kk': KazakhAnalyzer,
+    'uz': UzbekAnalyzer,
+    'mn': MongolianAnalyzer,
+
+    # Universal fallback
+    'xx': UniversalFallbackAnalyzer,
+}
+
+# Register all analyzers
+for code, analyzer_class in ANALYZERS.items():
+    AnalyzerRegistry.register(code, analyzer_class)
+
+
+def get_analyzer(lang: str = None, text: str = None, fallback: bool = True) -> AdvancedMorphologicalAnalyzer:
+    """
+    언어 코드로 분석기 인스턴스 생성
+
+    Args:
+        lang: ISO 639-1 언어 코드 (ko, ja, zh, en, de, fr, es, ru, ar, hi, ...)
+        text: 분석할 텍스트 (언어 자동 감지용)
+        fallback: True면 미지원 언어에 대해 UniversalFallback 반환
+
+    Returns:
+        해당 언어의 형태소 분석기 인스턴스
+
+    Raises:
+        ValueError: fallback=False이고 지원하지 않는 언어 코드일 때
+    """
+    if text and not lang:
+        # Auto-detect from text
+        return get_analyzer_auto(text=text)
+
+    if lang:
+        lang = lang.lower()
+        if lang in ANALYZERS:
+            return ANALYZERS[lang]()
+        elif fallback:
+            return UniversalFallbackAnalyzer()
+        else:
+            raise ValueError(f"Unsupported language: {lang}. Supported: {list(ANALYZERS.keys())}")
+
+    return UniversalFallbackAnalyzer()
+
+
+def supported_languages() -> list:
+    """지원 언어 목록 반환"""
+    return [code for code in ANALYZERS.keys() if code != 'xx']
+
+
+def language_info() -> dict:
+    """언어별 상세 정보 반환"""
+    return {
+        'tier1': ['ko', 'ja', 'zh', 'en', 'de', 'fr', 'es', 'ru', 'ar', 'hi'],
+        'tier2': ['pt', 'it', 'nl', 'pl', 'tr', 'vi', 'th', 'id', 'he', 'fa', 'uk', 'el', 'cs', 'ro', 'sv'],
+        'tier3': ['no', 'da', 'fi', 'hu', 'bg', 'hr', 'sr', 'sk', 'sl', 'lt', 'lv', 'et', 'ca', 'eu', 'gl', 'cy', 'ga', 'is', 'mt', 'sq', 'mk'],
+        'tier4_south_asian': ['bn', 'ta', 'te', 'mr', 'gu', 'kn', 'ml', 'pa', 'ur', 'ne', 'si'],
+        'tier4_southeast_asian': ['my', 'km', 'lo', 'tl'],
+        'tier4_african': ['sw', 'am', 'yo', 'ha', 'zu', 'af'],
+        'tier4_caucasian': ['ka', 'hy', 'az'],
+        'tier4_central_asian': ['kk', 'uz', 'mn'],
+        'total': len(ANALYZERS) - 1,  # Exclude 'xx'
+    }
+
+
+# =============================================================================
+# Unified API (recommended)
+# =============================================================================
+from .unified import (
+    UnifiedMorphAnalyzer,
+    LemmaResult,
+    get_unified_analyzer,
+    unified_supported_languages,
+    unified_language_info,
+    lemmatize,
+    analyze,
+)
+
+__all__ = [
+    # Unified API (recommended)
+    'lemmatize',           # lemmatize('running', 'en') -> 'run'
+    'analyze',             # analyze('text', 'en') -> [LemmaResult, ...]
+    'get_unified_analyzer',
+    'unified_supported_languages',
+    'unified_language_info',
+    'UnifiedMorphAnalyzer',
+    'LemmaResult',
+
+    # Base classes
+    'AdvancedMorphologicalAnalyzer',
+    'Morpheme',
+    'AnalysisResult',
+    'NBestResult',
+    'Domain',
+
+    # Factory (legacy)
+    'get_analyzer',
+    'supported_languages',
+    'language_info',
+    'ANALYZERS',
+    'AnalyzerRegistry',
+
+    # Universal Fallback
+    'UniversalFallbackAnalyzer',
+
+    # Tier 1 - Advanced
+    'KoreanAdvancedAnalyzer', 'KoreanAnalyzer',
+    'JapaneseAdvancedAnalyzer', 'JapaneseAnalyzer',
+    'ChineseAdvancedAnalyzer', 'ChineseAnalyzer',
+    'EnglishAdvancedAnalyzer', 'EnglishAnalyzer',
+    'GermanAdvancedAnalyzer', 'GermanAnalyzer',
+    'FrenchAdvancedAnalyzer', 'FrenchAnalyzer',
+    'SpanishAdvancedAnalyzer', 'SpanishAnalyzer',
+    'RussianAdvancedAnalyzer', 'RussianAnalyzer',
+    'ArabicAdvancedAnalyzer', 'ArabicAnalyzer',
+    'HindiAdvancedAnalyzer', 'HindiAnalyzer',
+
+    # Tier 2
+    'PortugueseAnalyzer', 'ItalianAnalyzer', 'DutchAnalyzer',
+    'PolishAnalyzer', 'TurkishAnalyzer', 'VietnameseAnalyzer',
+    'ThaiAnalyzer', 'IndonesianAnalyzer', 'HebrewAnalyzer',
+    'PersianAnalyzer', 'UkrainianAnalyzer', 'GreekAnalyzer',
+    'CzechAnalyzer', 'RomanianAnalyzer', 'SwedishAnalyzer',
+
+    # Tier 3
+    'NorwegianAnalyzer', 'DanishAnalyzer', 'FinnishAnalyzer',
+    'HungarianAnalyzer', 'BulgarianAnalyzer', 'CroatianAnalyzer',
+    'SerbianAnalyzer', 'SlovakAnalyzer', 'SlovenianAnalyzer',
+    'LithuanianAnalyzer', 'LatvianAnalyzer', 'EstonianAnalyzer',
+    'CatalanAnalyzer', 'BasqueAnalyzer', 'GalicianAnalyzer',
+    'WelshAnalyzer', 'IrishAnalyzer', 'IcelandicAnalyzer',
+    'MalteseAnalyzer', 'AlbanianAnalyzer', 'MacedonianAnalyzer',
+
+    # Tier 4 - South Asian
+    'BengaliAnalyzer', 'TamilAnalyzer', 'TeluguAnalyzer',
+    'MarathiAnalyzer', 'GujaratiAnalyzer', 'KannadaAnalyzer',
+    'MalayalamAnalyzer', 'PunjabiAnalyzer', 'UrduAnalyzer',
+    'NepaliAnalyzer', 'SinhalaAnalyzer',
+
+    # Tier 4 - Southeast Asian
+    'MyanmarAnalyzer', 'KhmerAnalyzer', 'LaoAnalyzer', 'TagalogAnalyzer', 'MalayAnalyzer',
+
+    # Tier 4 - African
+    'SwahiliAnalyzer', 'AmharicAnalyzer', 'YorubaAnalyzer',
+    'HausaAnalyzer', 'ZuluAnalyzer', 'AfrikaansAnalyzer',
+
+    # Tier 4 - Caucasian
+    'GeorgianAnalyzer', 'ArmenianAnalyzer', 'AzerbaijaniAnalyzer',
+
+    # Tier 4 - Central Asian
+    'KazakhAnalyzer', 'UzbekAnalyzer', 'MongolianAnalyzer',
+]
+
+__version__ = '3.0.0'