tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/unified.py

@@ -0,0 +1,855 @@
+"""
+Unified Morphological Analyzer
+==============================
+
+두 가지 형태소 시스템을 통합:
+1. 특화 분석기 (74개 언어) - 규칙 기반
+2. (옵션) lemma lexicon (언어별) - 사전 기반 (외부 자산)
+
+Note:
+- TokMor OSS core는 대형 lemma 사전을 wheel/sdist에 포함하지 않는다.
+- lemma lexicon은 필요하면 `TOKMOR_DATA_DIR/lemma_dict/` 또는 `TOKMOR_LEMMA_DICT_DIR`로 제공한다.
+"""
+
+import pickle
+import os
+from pathlib import Path
+from typing import Optional, List, Dict, Any, Tuple
+from dataclasses import dataclass
+
+from .advanced_base import AdvancedMorphologicalAnalyzer, Morpheme, AnalysisResult
+from . import ANALYZERS
+from .. import resources
+from ..lemma_store import BaseLemmaStore, load_lemma_store
+
+
+@dataclass
+class LemmaResult:
+    """Lemmatization result"""
+    word: str
+    lemma: str
+    pos: Optional[str] = None
+    features: Optional[str] = None
+    confidence: float = 1.0
+    source: str = 'unknown'  # 'lexicon', 'specialized', 'model', 'rule', 'fallback'
+
+
+class UnifiedMorphAnalyzer:
+    """
+    통합 형태소 분석기
+
+    우선순위:
+    1. 특화 분석기 (있으면)
+    2. lemma lexicon (있으면)
+    3. 규칙 기반 폴백
+    """
+
+    def __init__(self, lang: str):
+        self.lang = lang.lower()
+        self.specialized_analyzer = None
+        self._lemma_store: Optional[BaseLemmaStore] = None  # lazy-loaded (sqlite/pkl)
+        self._load_analyzers()
+
+    def _load_analyzers(self):
+        """분석기 로드"""
+        # 1. 특화 분석기 시도
+        if self.lang in ANALYZERS:
+            try:
+                self.specialized_analyzer = ANALYZERS[self.lang]()
+            except Exception:
+                pass
+
+    def _get_lemma_store(self) -> Optional[BaseLemmaStore]:
+        """
+        Load lemma lexicon store (surface -> lemma) if available.
+
+        Location (priority):
+          - TOKMOR_LEMMA_DICT_DIR/{lang}.sqlite|.db|.pkl ...
+          - tokmor/models/lemma_dict/{lang}.sqlite|.db|.pkl ...
+          - (legacy) some deployments used additional lemma packs under TOKMOR_DATA_DIR
+        """
+        if self._lemma_store is not None:
+            return self._lemma_store
+
+        # Allow lemma aliasing for top100 wiki-style codes (dev convenience)
+        lex_lang = resources.normalize_lang_for_lemma(self.lang) if hasattr(resources, "normalize_lang_for_lemma") else self.lang
+        lex_path = resources.resolve_lemma_dict_path(lex_lang)
+        if not lex_path:
+            self._lemma_store = None
+            return None
+
+        try:
+            self._lemma_store = load_lemma_store(lex_path)
+            return self._lemma_store
+        except Exception:
+            pass
+
+        self._lemma_store = None
+        return None
+
+    @property
+    def source(self) -> str:
+        """분석기 소스 반환"""
+        if self.specialized_analyzer:
+            return 'specialized'
+        return 'fallback'
+
+    @property
+    def is_available(self) -> bool:
+        """분석 가능 여부"""
+        # Quality spec for TokMor preprocessing:
+        # - segmentation is handled by tokenizer
+        # - lemma is always available at least as an identity fallback
+        # Therefore morphology should be "available" for any tokenized language.
+        return True
+
+    def lemmatize(self, word: str, pos_hint: Optional[str] = None) -> str:
+        """
+        단어의 기본형(lemma) 반환
+
+        Args:
+            word: 입력 단어
+
+        Returns:
+            기본형 (lemma)
+        """
+        result = self.lemmatize_detailed(word, pos_hint=pos_hint)
+        return result.lemma
+
+    def lemmatize_detailed(self, word: str, *, pos_hint: Optional[str] = None) -> LemmaResult:
+        """
+        상세 lemmatization 결과 반환
+
+        Args:
+            word: 입력 단어
+
+        Returns:
+            LemmaResult with lemma, pos, features, source
+        """
+        word_lower = word.lower()
+
+        # 0. Lemma lexicon (if present): highest priority
+        store = self._get_lemma_store()
+        if store:
+            # Latin-script languages: store lowercase key; otherwise keep exact surface too.
+            key = word_lower if self.lang in {"en", "de", "fr", "es", "it", "pt", "nl", "sv", "da", "no", "fi"} else word
+            lemma = store.get(key)
+            if lemma:
+                # Some legacy lexicons store decomposed lemmas like "먹+었+다".
+                # For Korean, prefer the specialized analyzer unless the lexicon provides a clean lemma.
+                if self.lang == "ko" and isinstance(lemma, str) and "+" in lemma:
+                    style = os.getenv("TOKMOR_KO_LEMMA_STYLE", "dict").strip().lower()
+                    if style not in ("decomp",):
+                        lemma = None
+                if lemma:
+                    if self.lang == "en":
+                        lemma = self._en_postprocess_lemma(word, lemma, source="lexicon", pos_hint=pos_hint)
+                    return LemmaResult(
+                        word=word,
+                        lemma=lemma,
+                        confidence=0.99,
+                        source="lexicon",
+                    )
+
+        # Pack-less mode: prefer conservative fallback rules across languages.
+        # Many "advanced analyzers" are dictionary/heuristic segmenters and can output partial lemmas
+        # (worse than a stable normalized surface) when no lemma pack is present.
+        packless = os.getenv("TOKMOR_DISABLE_LEMMA_PACK", "").strip().lower() in {"1", "true", "yes", "y", "on"}
+
+        # 1. 특화 분석기 시도
+        # NOTE: for Chinese, lemma is effectively identity; specialized segmentation can produce
+        # undesirable "partial" lemmas for combined tokens (e.g., dates). Prefer fallback.
+        if self.lang == "zh" or (packless and self.lang not in {"ko"}):
+            self.specialized_analyzer = None
+
+        if self.specialized_analyzer:
+            try:
+                result = self.specialized_analyzer.analyze(word)
+
+                # Advanced analyzers often return NBestResult -> use .best.morphemes
+                morphemes = None
+                confidence = 1.0
+                if hasattr(result, "best") and hasattr(result.best, "morphemes"):
+                    morphemes = result.best.morphemes
+                    confidence = getattr(result.best, "score", 1.0)
+                elif hasattr(result, "morphemes"):
+                    morphemes = result.morphemes
+                    confidence = getattr(result, "score", 1.0)
+
+                if morphemes:
+                    # Korean: reconstruct lemma for verb/adjective and noun+XSV(하다) patterns
+                    if self.lang == "ko":
+                        lemma, pos = self._ko_reconstruct_lemma(morphemes)
+                        return LemmaResult(
+                            word=word,
+                            lemma=lemma,
+                            pos=pos,
+                            features=getattr(morphemes[0], "features", None),
+                            confidence=float(confidence),
+                            source="specialized",
+                        )
+
+                    # Japanese: avoid returning a function morpheme as "lemma" (e.g., もの -> も).
+                    if self.lang == "ja":
+                        # Prefer a "content" morpheme lemma; fallback to normalized surface.
+                        def _is_ja_function_pos(p: Optional[str]) -> bool:
+                            if not p:
+                                return False
+                            # Common function tags in our JA analyzer family
+                            if p.startswith("助"):
+                                return True  # 助詞/助動詞...
+                            if p in {"連用形", "終止形", "未然形", "促音便"}:
+                                return True
+                            return False
+
+                        for m in morphemes:
+                            mp = getattr(m, "pos", None)
+                            if _is_ja_function_pos(mp):
+                                continue
+                            lem0 = getattr(m, "lemma", getattr(m, "form", None))
+                            if isinstance(lem0, str) and lem0:
+                                return LemmaResult(
+                                    word=word,
+                                    lemma=lem0,
+                                    pos=mp,
+                                    features=getattr(m, "features", None),
+                                    confidence=float(confidence),
+                                    source="specialized",
+                                )
+                        # If everything looked like function parts, fall back to rule-based path.
+                        raise RuntimeError("ja specialized produced only function morphemes")
+
+                    # Default: first morpheme lemma
+                    m0 = morphemes[0]
+                    lem = getattr(m0, "lemma", getattr(m0, "form", word))
+                    if self.lang == "en":
+                        lem = self._en_postprocess_lemma(word, str(lem), source="specialized", pos_hint=pos_hint)
+                    return LemmaResult(
+                        word=word,
+                        lemma=lem,
+                        pos=getattr(m0, "pos", None),
+                        features=getattr(m0, "features", None),
+                        confidence=float(confidence),
+                        source="specialized",
+                    )
+            except Exception:
+                pass
+
+        # 2. 규칙 기반 폴백
+        lemma = self._fallback_lemmatize(word_lower, original=word)
+        if self.lang == "en":
+            lemma = self._en_postprocess_lemma(word, lemma, source="fallback", pos_hint=pos_hint)
+        return LemmaResult(
+            word=word,
+            lemma=lemma,
+            confidence=0.3,
+            source='fallback'
+        )
+
+    def _en_postprocess_lemma(self, word: str, lemma: str, *, source: str = "", pos_hint: Optional[str] = None) -> str:
+        """
+        English lemma postprocessing to better match common corpus conventions on noisy/web corpora.
+        This runs AFTER lexicon lookup and AFTER rule-based fallback.
+        """
+        import unicodedata
+
+        w = word or ""
+        wl = w.lower()
+        lem = lemma or w
+
+        # 1) Common clitics / contractions (token-level, no context)
+        # Note: "'s" is ambiguous (POS=PART vs AUX). In EWT test, "'s" is slightly more common.
+        CONTR = {
+            "n't": "not",
+            "'re": "be",
+            "'m": "be",
+            "'ve": "have",
+            "'ll": "will",
+            "'d": "would",
+            # "'s" is ambiguous: AUX -> be/have, PART -> 's
+            "'s": "'s",
+        }
+        if wl in CONTR:
+            if wl == "'s":
+                # Use coarse POS hint when available: AUX/VERB-like -> be
+                ph = (pos_hint or "").upper()
+                if ph in {"V"}:
+                    return "be"
+            return CONTR[wl]
+        # Token "s" (missing apostrophe in noisy text) is ambiguous; in EWT it most often maps to BE.
+        if wl == "s" and w == "s":
+            ph = (pos_hint or "").upper()
+            if ph in {"V"}:
+                return "be"
+            return "'s"
+
+        # 2) Preserve punctuation-only tokens exactly (..., --, etc.)
+        def _is_punct_or_symbol_only(s: str) -> bool:
+            return bool(s) and all(unicodedata.category(ch)[0] in {"P", "S"} for ch in s)
+
+        if _is_punct_or_symbol_only(w):
+            return w
+
+        # 3) Preserve special mixed tokens (dates, ids, emails, domains, filenames, phone numbers)
+        has_digit = any(ch.isdigit() for ch in w)
+        has_alpha = any(ch.isalpha() for ch in w)
+        has_special_sep = any(ch in {"@", ".", "-", "_", "/", ":"} for ch in w)
+        if (has_digit and (has_alpha or has_special_sep)) or (("@" in w) and has_alpha):
+            return w
+        # phone-like patterns (digits with separators)
+        if has_digit and any(ch in {"-", "/"} for ch in w):
+            return w
+
+        # 4) Abbreviations (web/news specific wins)
+        if w == "AM":
+            return "a.m."
+        if w == "PM":
+            return "p.m."
+        # Abbreviations ending with a dot: keep surface (Inc., Dr., U.S., D.C., ...)
+        if "." in w and w.endswith(".") and any(ch.isalpha() for ch in w):
+            return w
+
+        # 5) Acronyms / all-caps
+        if len(w) >= 2 and w.isupper() and w.isalpha():
+            # Short 2-letter country/region acronyms are usually kept as-is (US, UK, EU, ...).
+            if len(w) <= 2:
+                return w
+            # If lexicon produced a lemma, trust it (common words often appear in all-caps in headlines).
+            if source == "lexicon":
+                return lem
+            # Heuristic: vowel-less all-caps tokens are often technical acronyms -> lowercase (MMBTU -> mmbtu)
+            vcnt = sum(1 for ch in w if ch in "AEIOU")
+            if vcnt <= 1:
+                return wl
+            # Otherwise, likely a name written in all-caps -> Titlecase (SOUTER -> Souter)
+            return w.title()
+
+        # 6) Emoticons with letters: lowercase (e.g., :D -> :d)
+        if any(ch.isalpha() for ch in w) and sum(ch.isalnum() for ch in w) <= 2 and has_special_sep:
+            return w.lower()
+
+        # 7) Case normalization fixes
+        # If lexicon returns a titlecased lemma for a lowercase token, prefer the lowercase surface.
+        if w.islower() and len(lem) > 1 and lem[0].isupper() and lem.lower() == wl:
+            return w
+
+        # If token is Titlecase and lemma is the fully-lowercased form, it can be a PROPN-like lemma.
+        # Apply only for non-lexicon sources to avoid harming normal sentence-initial capitalization.
+        if source != "lexicon" and len(w) >= 2 and w[0].isupper() and w[1:].islower() and lem == wl:
+            TITLE_STOP = {
+                "i", "a", "an", "the", "this", "that", "these", "those",
+                "and", "or", "but", "if", "in", "on", "at", "to", "of", "for", "from", "by", "with", "as",
+                "do", "did", "does", "is", "are", "was", "were", "be",
+                "my", "your", "our", "their", "his", "her",
+                "yes", "no", "hi", "hello", "thanks", "thank", "please", "what", "how", "why", "when", "where", "who",
+                "good", "great", "very", "email",
+            }
+            if wl not in TITLE_STOP:
+                return w
+
+        # 8) Pronoun lemma overrides (unambiguous subset)
+        PRON = {
+            "us": "we",
+            "me": "I",
+            "him": "he",
+            "them": "they",
+        }
+        if wl in PRON and w.islower():
+            return PRON[wl]
+
+        return lem
+
+    def _ko_reconstruct_lemma(self, morphemes) -> Tuple[str, Optional[str]]:
+        """
+        Korean lemma reconstruction from morpheme sequence.
+        Goal: produce dictionary-form lemma closer to common usage (e.g., 먹었다 -> 먹다, 했다 -> 하다, 발표했다 -> 발표하다).
+        """
+        def _pos(m):
+            return getattr(m, "pos", "") or ""
+
+        def _lem(m):
+            return getattr(m, "lemma", getattr(m, "form", getattr(m, "surface", ""))) or ""
+
+        # 1) If there's an explicit verb/adjective stem, use it + '다'
+        for m in morphemes:
+            p = _pos(m)
+            if p in {"VV", "VA", "VX", "VCP", "VCN"}:
+                stem = _lem(m)
+                if stem:
+                    return (stem + "다", p)
+
+        # 2) Noun + XSV('하') pattern -> noun + '하다'
+        if len(morphemes) >= 2:
+            m0, m1 = morphemes[0], morphemes[1]
+            if _pos(m0).startswith("N") and _pos(m1) == "XSV" and _lem(m1) == "하":
+                n = _lem(m0)
+                if n:
+                    return (n + "하다", "VV")
+
+        # 3) Fallback: first morpheme lemma
+        m0 = morphemes[0]
+        return (_lem(m0) or word, _pos(m0) or None)
+
+    def _apply_suffix_rules(self, word: str, rules: Dict) -> Optional[Dict]:
+        """접미사 규칙 적용"""
+        best_match = None
+        best_length = 0
+
+        for suffix, rule in rules.items():
+            if word.endswith(suffix) and len(suffix) > best_length:
+                strip = rule.get('strip', len(suffix))
+                add = rule.get('add', '')
+
+                if len(word) > strip:
+                    lemma = word[:-strip] + add if strip > 0 else word + add
+                    if lemma:  # 빈 문자열 방지
+                        best_match = {
+                            'lemma': lemma,
+                            'features': rule.get('features'),
+                            'prob': rule.get('prob', 0.5)
+                        }
+                        best_length = len(suffix)
+
+        return best_match
+
+    def _fallback_lemmatize(self, word: str, *, original: Optional[str] = None) -> str:
+        """
+        규칙 기반 폴백 lemmatization (pack-less / lexicon-less path)
+
+        목표:
+        - 어떤 언어/토큰이 와도 절대 크래시하지 않음
+        - 과도한 어간추출(stemming)로 의미 훼손하지 않음(보수적)
+        - 최소한의 정규화: NFC + 양끝 구두점 제거 + (가능하면) 소문자화
+        """
+        import unicodedata
+
+        w0 = original if original is not None else word
+        try:
+            w0 = unicodedata.normalize("NFC", w0)
+        except Exception:
+            pass
+        # Strip common edge punctuation (keep internal apostrophes/hyphens)
+        try:
+            w0 = w0.strip().strip(" \t\r\n")
+            # Include common CJK punctuation too (。！？、，：； etc.)
+            w0 = w0.strip(".,;:!?\"“”‘’`()[]{}<>«»。、，！？：；（）【】『』「」《》〈〉")
+        except Exception:
+            pass
+        if not w0:
+            return word
+
+        def _has_casing(s: str) -> bool:
+            # True if the script has case distinctions (Latin/Cyrillic/Greek/etc.).
+            # (For uncased scripts, lower/casefold is a no-op anyway, but keep it explicit.)
+            try:
+                for ch in s:
+                    if not ch.isalpha():
+                        continue
+                    if ch.lower() != ch.upper():
+                        return True
+            except Exception:
+                pass
+            return False
+
+        # Default: casefold for any cased script (handles Latin/Cyrillic/Greek/etc).
+        try:
+            wl = w0.casefold() if _has_casing(w0) else w0
+        except Exception:
+            wl = w0.lower() if w0.isascii() else w0
+
+        # Arabic-family normalization (keeps it conservative; improves pack-less matching/stability)
+        if self.lang in {"ar", "fa", "ur"}:
+            try:
+                # Remove tatweel and common harakat/diacritics
+                wl = wl.replace("ـ", "")
+                wl = "".join(ch for ch in wl if unicodedata.category(ch) != "Mn")
+                # Normalize alef variants + ya/maqsura (useful for ar; harmless for fa/ur)
+                wl = wl.translate(
+                    str.maketrans(
+                        {
+                            "أ": "ا",
+                            "إ": "ا",
+                            "آ": "ا",
+                            "ى": "ي",
+                        }
+                    )
+                )
+            except Exception:
+                pass
+
+        # Hebrew: strip niqqud (Mn marks)
+        if self.lang == "he":
+            try:
+                wl = "".join(ch for ch in wl if unicodedata.category(ch) != "Mn")
+            except Exception:
+                pass
+
+        # Turkic orthography: split case-marking after apostrophe in proper nouns (Ankara'ya -> ankara)
+        if self.lang in {"tr", "az", "kk", "uz"}:
+            if "'" in wl:
+                try:
+                    left, right = wl.split("'", 1)
+                    if left and right and left.isalpha() and right.isalpha():
+                        wl = left
+                except Exception:
+                    pass
+
+        # Romance elision clitics: l'amour -> amour (very conservative)
+        if self.lang in {"fr", "it", "ca"}:
+            if "'" in wl:
+                try:
+                    left, right = wl.split("'", 1)
+                    if 1 <= len(left) <= 2 and right and right[0].isalpha():
+                        wl = right
+                except Exception:
+                    pass
+
+        def _is_hangul_syllable(ch: str) -> bool:
+            o = ord(ch)
+            return 0xAC00 <= o <= 0xD7A3
+
+        def _ko_strip_josa(s: str) -> str:
+            # Conservative: strip only common postpositions/endings if token is Hangul and longer than suffix.
+            if not s or len(s) < 2:
+                return s
+            if not all(_is_hangul_syllable(c) for c in s):
+                return s
+            # Longest-first
+            suffixes = [
+                "으로부터",
+                "에서부터",
+                "으로써",
+                "로써",
+                "으로서",
+                "로서",
+                "에게서",
+                "까지",
+                "부터",
+                "에서",
+                "에게",
+                "한테",
+                "께서",
+                "께",
+                "보다",
+                "처럼",
+                "마저",
+                "조차",
+                "라도",
+                "이나",
+                "나",
+                "와",
+                "과",
+                "으로",
+                "로",
+                "에",
+                "의",
+                "도",
+                "만",
+                "은",
+                "는",
+                "이",
+                "가",
+                "을",
+                "를",
+            ]
+            for suf in suffixes:
+                if len(s) > len(suf) and s.endswith(suf):
+                    stem = s[: -len(suf)]
+                    if stem:
+                        return stem
+            return s
+
+        def _is_kanji(ch: str) -> bool:
+            o = ord(ch)
+            return (0x4E00 <= o <= 0x9FFF) or (0x3400 <= o <= 0x4DBF)
+
+        def _is_kana(ch: str) -> bool:
+            o = ord(ch)
+            return (0x3040 <= o <= 0x309F) or (0x30A0 <= o <= 0x30FF)
+
+        def _ja_strip_particles(s: str) -> str:
+            # Conservative: strip only if the token ends with a particle and the preceding char looks content-ish.
+            if not s or len(s) < 2:
+                return s
+            # multi-char first
+            multi = ["から", "まで", "より"]
+            for suf in multi:
+                if len(s) > len(suf) and s.endswith(suf):
+                    stem = s[: -len(suf)]
+                    if stem:
+                        return stem
+            # single-char particles: only strip if previous char is Kanji/Katakana (avoid stripping from pure-hiragana words like もの)
+            single = ["は", "が", "を", "に", "で", "と", "も", "へ", "の", "や", "か", "ね", "よ", "な"]
+            last = s[-1]
+            if last in single:
+                prev = s[-2]
+                if _is_kanji(prev) or (0x30A0 <= ord(prev) <= 0x30FF):
+                    stem = s[:-1]
+                    if stem:
+                        return stem
+            return s
+
+        # 영어 기본 규칙(기존 유지)
+        if self.lang == 'en':
+            # -ing
+            if wl.endswith('ing') and len(wl) > 4:
+                base = wl[:-3]
+                if base.endswith('e'):
+                    return base
+                if len(base) > 2 and base[-1] == base[-2]:  # running -> run
+                    return base[:-1]
+                return base + 'e' if base[-1] not in 'aeiou' else base
+            # -ed
+            if wl.endswith('ed') and len(wl) > 3:
+                base = wl[:-2]
+                if wl.endswith('ied'):
+                    return wl[:-3] + 'y'
+                if len(base) > 2 and base[-1] == base[-2]:  # stopped -> stop
+                    return base[:-1]
+                return base + 'e' if base[-1] not in 'aeiou' else base
+            # -s, -es
+            if wl.endswith('ies') and len(wl) > 3:
+                return wl[:-3] + 'y'
+            if wl.endswith('es') and len(wl) > 3:
+                return wl[:-2]
+            if wl.endswith('s') and len(wl) > 2:
+                return wl[:-1]
+            return wl
+
+        # High-value pack-less tuning: strip common function-word suffixes in agglutinative no-space-ish usage.
+        if self.lang == "ko":
+            return _ko_strip_josa(wl)
+        if self.lang == "ja":
+            return _ja_strip_particles(wl)
+
+        # Pack-less fallback for most languages: conservative normalization only
+        # - If ASCII: lowercase to stabilize (matches common lemma conventions for Latin-script langs)
+        # - Else: keep as-is (already NFC + edge punct stripped)
+        return wl
+
+    def analyze(self, text: str) -> List[LemmaResult]:
+        """
+        텍스트 전체 분석
+
+        Args:
+            text: 입력 텍스트
+
+        Returns:
+            LemmaResult 리스트
+        """
+        # SEA no-space scripts: the template analyzers tend to group whole runs,
+        # while TokMor tokenizers (Brahmic/Thai native) provide much better segmentation.
+        # For product stability, prefer tokenizer-backed segmentation here.
+        if self.lang in {"th", "km", "lo", "my"}:
+            try:
+                from ..factory import get_tokenizer  # lazy import
+                use_morph = (self.lang == "th")
+                tok = get_tokenizer(self.lang, use_morphology=use_morph)
+                tres = tok.tokenize(text)
+                out: List[LemmaResult] = []
+                for t in (tres.tokens or []):
+                    try:
+                        det = self.lemmatize_detailed(t.text)
+                        lemma = t.lemma if getattr(t, "lemma", None) else det.lemma
+                        out.append(
+                            LemmaResult(
+                                word=t.text,
+                                lemma=lemma,
+                                pos=getattr(t, "pos", None) or det.pos,
+                                features=str(getattr(t, "features", None)) if getattr(t, "features", None) else det.features,
+                                confidence=float(det.confidence),
+                                source=det.source,
+                            )
+                        )
+                    except Exception:
+                        out.append(LemmaResult(word=t.text, lemma=t.text, source="fallback", confidence=0.1))
+                if out:
+                    return out
+            except Exception:
+                # fall through to generic logic
+                pass
+
+        # 특화 분석기 있으면 사용
+        if self.specialized_analyzer:
+            try:
+                result = self.specialized_analyzer.analyze(text)
+                if result and result.morphemes:
+                    return [
+                        LemmaResult(
+                            word=m.form,
+                            lemma=getattr(m, 'lemma', m.form),
+                            pos=m.pos,
+                            features=getattr(m, 'features', None),
+                            source='specialized'
+                        )
+                        for m in result.morphemes
+                    ]
+            except Exception:
+                pass
+
+        # Generic fallback: tokenizer-backed segmentation (better than whitespace split for many scripts)
+        try:
+            from ..factory import get_tokenizer  # lazy import
+            tok = get_tokenizer(self.lang, use_morphology=False)
+            tokens = tok.tokenize(text).texts()
+            if tokens:
+                return [self.lemmatize_detailed(w) for w in tokens]
+        except Exception:
+            pass
+
+        # Last fallback: whitespace split
+        words = text.split()
+        return [self.lemmatize_detailed(w) for w in words if w]
+
+    def get_morphemes(self, word: str) -> List[Morpheme]:
+        """
+        단어의 형태소 분석 결과 반환 (특화 분석기 전용)
+
+        Args:
+            word: 입력 단어
+
+        Returns:
+            Morpheme 리스트
+        """
+        if self.specialized_analyzer:
+            try:
+                result = self.specialized_analyzer.analyze(word)
+                if result and result.morphemes:
+                    return result.morphemes
+            except Exception:
+                pass
+
+        # 폴백: 단일 형태소로 반환
+        lemma_result = self.lemmatize_detailed(word)
+        return [Morpheme(
+            form=word,
+            pos=lemma_result.pos or 'X',
+            lemma=lemma_result.lemma,
+            features=lemma_result.features
+        )]
+
+
+# 캐시
+_unified_analyzers: Dict[str, UnifiedMorphAnalyzer] = {}
+
+
+def get_unified_analyzer(lang: str) -> UnifiedMorphAnalyzer:
+    """
+    통합 형태소 분석기 반환
+
+    Args:
+        lang: 언어 코드
+
+    Returns:
+        UnifiedMorphAnalyzer 인스턴스
+    """
+    lang = lang.lower()
+
+    if lang not in _unified_analyzers:
+        _unified_analyzers[lang] = UnifiedMorphAnalyzer(lang)
+
+    return _unified_analyzers[lang]
+
+
+def unified_supported_languages() -> List[str]:
+    """
+    통합 형태소 분석 지원 언어 목록
+
+    Returns:
+        언어 코드 리스트 (특화 + lemma lexicon 합집합)
+    """
+    # 특화 분석기 언어
+    specialized = set(ANALYZERS.keys()) - {'xx'}
+    # Tokenize-supported languages (fallback lemma is always available)
+    tokenize_langs = set()
+    try:
+        from ..factory import supported_languages as _supported_languages  # lazy import
+        tokenize_langs = set(_supported_languages())
+    except Exception:
+        tokenize_langs = set()
+    lex_langs = set()
+    try:
+        ld = resources.lemma_dict_dir()
+        if ld.exists():
+            for f in ld.glob("*"):
+                if f.suffix.lower() in (".pkl", ".sqlite", ".db", ".sqlite3"):
+                    lex_langs.add(f.stem.split("_")[0])
+    except Exception:
+        pass
+    # Legacy external lemma packs are intentionally not supported in OSS distribution.
+
+    return sorted(tokenize_langs | specialized | lex_langs)
+
+
+def unified_language_info() -> Dict[str, Any]:
+    """
+    통합 형태소 분석 언어별 상세 정보
+
+    Returns:
+        언어별 정보 딕셔너리
+    """
+    # 특화 분석기 언어
+    specialized = set(ANALYZERS.keys()) - {'xx'}
+
+    # lemma lexicon 언어
+    lex_langs = set()
+    try:
+        ld = resources.lemma_dict_dir()
+        if ld.exists():
+            for f in ld.glob("*"):
+                if f.suffix.lower() in (".pkl", ".sqlite", ".db", ".sqlite3"):
+                    lex_langs.add(f.stem.split("_")[0])
+    except Exception:
+        pass
+    # Legacy external lemma packs are intentionally not supported in OSS distribution.
+
+    # 언어별 정보 조합
+    all_langs = specialized | set(lex_langs)
+
+    info = {
+        'total': len(all_langs),
+        'specialized_count': len(specialized),
+        'lexicon_count': len(lex_langs),
+        'overlap_count': len(specialized & set(lex_langs)),
+        'languages': {}
+    }
+
+    for lang in sorted(all_langs):
+        lang_info = {
+            'has_specialized': lang in specialized,
+            'has_lexicon': lang in lex_langs
+        }
+        info['languages'][lang] = lang_info
+
+    return info
+
+
+# 편의 함수
+def lemmatize(word: str, lang: str = 'en') -> str:
+    """
+    단어의 기본형 반환 (편의 함수)
+
+    Args:
+        word: 입력 단어
+        lang: 언어 코드
+
+    Returns:
+        기본형 (lemma)
+    """
+    analyzer = get_unified_analyzer(lang)
+    return analyzer.lemmatize(word)
+
+
+def analyze(text: str, lang: str = 'en') -> List[LemmaResult]:
+    """
+    텍스트 형태소 분석 (편의 함수)
+
+    Args:
+        text: 입력 텍스트
+        lang: 언어 코드
+
+    Returns:
+        LemmaResult 리스트
+    """
+    analyzer = get_unified_analyzer(lang)
+    return analyzer.analyze(text)