tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/chinese.py

@@ -0,0 +1,736 @@
+"""
+Chinese Morphological Analyzer - 자체 구현
+=========================================
+
+외부 라이브러리 없이 순수 Python으로 구현한 중국어 분석기
+
+특징:
+- 최장일치 분词 (Maximum Matching)
+- 역방향 최장일치 (Reverse Maximum Matching)
+- 양방향 비교로 최적 분할 선택
+
+"""
+
+import os
+import re
+import math
+import pickle
+from typing import List, Tuple, Set, Optional
+from dataclasses import dataclass
+
+from ..resources import resolve_seg_lexicon_path, resolve_extra_dict_path
+
+
+@dataclass
+class Morpheme:
+    """형태소/단어"""
+    surface: str
+    lemma: str
+    pos: str
+    start: int
+    end: int
+
+    def __repr__(self):
+        return f"{self.surface}/{self.pos}"
+
+
+class ChineseAnalyzer:
+    """
+    중국어 분석기
+
+    Usage:
+        analyzer = ChineseAnalyzer()
+        result = analyzer.analyze("阿里巴巴集团在杭州宣布")
+    """
+
+    def __init__(self, join_dates: Optional[bool] = None):
+        # Product option: join common date spans into a single token for news/SNS.
+        # Default is OFF to avoid unexpected evaluation/style mismatches.
+        if join_dates is None:
+            self.join_dates = os.getenv("TOKMOR_ZH_JOIN_DATES", "0").strip().lower() in ("1", "true", "yes", "y", "on")
+        else:
+            self.join_dates = bool(join_dates)
+        # Optional segmentation lexicon (offline, generated from large corpora):
+        # {word(str): freq(int)}. If present, we use a DP/Viterbi segmenter for Hanzi runs.
+        self._wordfreq = None
+        self._wordfreq_max_len = 4
+        self._build_dictionary()
+        self._load_seg_lexicon()
+        self._load_extra_dict()
+
+    def _load_extra_dict(self) -> None:
+        """
+        Optional runtime extension dictionary (offline).
+        File: seg_lexicon/zh_extra_dict.json  (token -> pos)
+        """
+        p = resolve_extra_dict_path("zh")
+        if not p:
+            return
+        try:
+            import json
+
+            obj = json.loads(p.read_text(encoding="utf-8", errors="ignore"))
+            if not isinstance(obj, dict):
+                return
+            # Merge into dictionary; keep only sane entries.
+            for k, v in obj.items():
+                if not isinstance(k, str) or not k:
+                    continue
+                if not isinstance(v, str) or not v:
+                    v = "n"
+                self.dictionary[k] = v
+            # Update max word length
+            self.max_word_len = max(len(w) for w in self.dictionary) if self.dictionary else self.max_word_len
+        except Exception:
+            return
+
+    def _load_seg_lexicon(self):
+        p = resolve_seg_lexicon_path("zh")
+        if not p:
+            return
+        try:
+            obj = pickle.loads(p.read_bytes())
+            if isinstance(obj, dict):
+                # keep only str->int like entries
+                wf = {}
+                mx = 1
+                for k, v in obj.items():
+                    if isinstance(k, str) and isinstance(v, int) and k:
+                        wf[k] = v
+                        if len(k) > mx:
+                            mx = len(k)
+                self._wordfreq = wf
+                self._wordfreq_max_len = max(2, min(int(mx), 8))
+        except Exception:
+            return
+
+    def _build_dictionary(self):
+        """사전 구축"""
+
+        # 주요 단어 사전 (단어: 품사)
+        self.dictionary = {
+            # ============================================================
+            # News / SNS domain (high-impact for segmentation)
+            # ============================================================
+            # 통신사/매체
+            '新华社': 'nrt', '中新社': 'nrt', '人民网': 'nrt', '央视': 'nrt', '央视网': 'nrt',
+            '澎湃新闻': 'nrt', '环球时报': 'nrt', '财新': 'nrt', '凤凰网': 'nrt', '网易': 'nrt', '新浪': 'nrt',
+            # 뉴스 관용구
+            '日电': 'n', '日讯': 'n', '消息': 'n', '报道': 'n', '记者': 'n',
+            '原标题': 'n', '来源': 'n', '编辑': 'n', '评论': 'n',
+            # 뉴스 단골(분절 안정화)
+            '直击': 'v', '强震': 'n', '灾区': 'n',
+            # SNS
+            '微博': 'n', '微信': 'n', '抖音': 'n', '快手': 'n', '小红书': 'n',
+            '网友': 'n', '点赞': 'v', '转发': 'v', '评论区': 'n',
+
+            # 지명
+            '北京': 'ns', '上海': 'ns', '广州': 'ns', '深圳': 'ns',
+            '杭州': 'ns', '南京': 'ns', '武汉': 'ns', '成都': 'ns',
+            '香港': 'ns', '台湾': 'ns', '中国': 'ns', '美国': 'ns',
+            '日本': 'ns', '韩国': 'ns', '英国': 'ns', '法国': 'ns',
+            '德国': 'ns', '俄罗斯': 'ns', '印度': 'ns', '越南': 'ns',
+            '泰国': 'ns', '新加坡': 'ns', '马来西亚': 'ns',
+            '土耳其': 'ns',
+
+            # 기업/조직
+            '阿里巴巴': 'nrt', '腾讯': 'nrt', '百度': 'nrt', '华为': 'nrt',
+            '小米': 'nrt', '京东': 'nrt', '美团': 'nrt', '字节跳动': 'nrt',
+            '苹果': 'nrt', '谷歌': 'nrt', '微软': 'nrt', '三星': 'nrt',
+            '特斯拉': 'nrt', '丰田': 'nrt', '奔驰': 'nrt', '宝马': 'nrt',
+            '集团': 'n', '公司': 'n', '企业': 'n', '银行': 'n',
+            '政府': 'n', '学校': 'n', '大学': 'n', '医院': 'n',
+
+            # 일반명사
+            '人': 'n', '人们': 'n', '时间': 'n', '地方': 'n',
+            '问题': 'n', '情况': 'n', '工作': 'n', '生活': 'n',
+            '发展': 'n', '经济': 'n', '社会': 'n', '文化': 'n',
+            '技术': 'n', '产品': 'n', '服务': 'n', '市场': 'n',
+            '国家': 'n', '世界': 'n', '城市': 'n', '地区': 'n',
+
+            # 동사
+            '是': 'v', '有': 'v', '在': 'v', '说': 'v', '做': 'v',
+            '去': 'v', '来': 'v', '看': 'v', '想': 'v', '知道': 'v',
+            '发表': 'v', '宣布': 'v', '公布': 'v', '发布': 'v',
+            '开始': 'v', '结束': 'v', '进行': 'v', '完成': 'v',
+            '研究': 'v', '开发': 'v', '生产': 'v', '销售': 'v',
+            '投资': 'v', '合作': 'v', '成立': 'v', '成为': 'v',
+
+            # 형용사
+            '大': 'a', '小': 'a', '多': 'a', '少': 'a',
+            '好': 'a', '新': 'a', '高': 'a', '重要': 'a',
+
+            # 부사
+            '不': 'd', '也': 'd', '就': 'd', '都': 'd', '还': 'd',
+            '很': 'd', '最': 'd', '已经': 'd', '正在': 'd',
+
+            # 개사 (전치사)
+            '在': 'p', '从': 'p', '向': 'p', '对': 'p', '把': 'p',
+            '被': 'p', '比': 'p', '跟': 'p', '和': 'p', '与': 'p',
+
+            # 조사/어기사
+            '的': 'u', '地': 'u', '得': 'u', '了': 'u', '着': 'u', '过': 'u',
+            '吗': 'u', '呢': 'u', '吧': 'u', '啊': 'u',
+
+            # 대명사
+            '我': 'r', '你': 'r', '他': 'r', '她': 'r', '它': 'r',
+            '我们': 'r', '你们': 'r', '他们': 'r', '这': 'r', '那': 'r',
+            '这个': 'r', '那个': 'r', '什么': 'r', '谁': 'r',
+
+            # 연결어
+            '和': 'c', '或': 'c', '但': 'c', '但是': 'c', '因为': 'c',
+            '所以': 'c', '如果': 'c', '虽然': 'c',
+
+            # 수사
+            '一': 'm', '二': 'm', '三': 'm', '四': 'm', '五': 'm',
+            '六': 'm', '七': 'm', '八': 'm', '九': 'm', '十': 'm',
+            '百': 'm', '千': 'm', '万': 'm', '亿': 'm',
+
+            # 양사
+            '个': 'q', '年': 'q', '月': 'q', '日': 'q', '号': 'q',
+            '次': 'q', '种': 'q', '件': 'q', '位': 'q',
+        }
+
+        # 최대 단어 길이
+        self.max_word_len = max(len(w) for w in self.dictionary) if self.dictionary else 4
+
+        # 한자 패턴
+        self.hanzi = re.compile(r'[\u4e00-\u9fff]+')
+
+        # 날짜/시간 패턴 (뉴스에서 매우 자주 등장)
+        # Examples: 2025年12月31日, 12月31日, 12月31日电
+        # NOTE: only used when self.join_dates is True
+        self._date_ymd = re.compile(r'^[0-9]{2,4}年[0-9]{1,2}月[0-9]{1,2}[日号]')
+        self._date_md = re.compile(r'^[0-9]{1,2}月[0-9]{1,2}[日号]')
+        self._date_md_with_dian = re.compile(r'^[0-9]{1,2}月[0-9]{1,2}[日号]电')
+
+    def analyze(self, text: str) -> List[Morpheme]:
+        """
+        Analyze text into morphemes.
+        """
+        if not text:
+            return []
+
+        # If we have a segmentation lexicon, prefer DP segmentation (more stable on real corpora).
+        if self._wordfreq:
+            out = self._segment_with_lexicon(text)
+            out = self._postprocess_fix_mixed_function_word_tokens(out)
+            return self._postprocess_merge_common_suffixes(out)
+
+        # Fallback: 정방향/역방향 최장일치
+        forward = self._forward_max_match(text)
+        backward = self._backward_max_match(text)
+        out = forward if len(forward) <= len(backward) else backward
+        out = self._postprocess_fix_mixed_function_word_tokens(out)
+        return self._postprocess_merge_common_suffixes(out)
+
+    def _postprocess_fix_mixed_function_word_tokens(self, toks: List[Morpheme]) -> List[Morpheme]:
+        """
+        Fix a common segmentation error from n-gram lexicons:
+        - function word + first char of a toponym gets merged (e.g., 在北 + 京市)
+
+        We do a conservative local split/merge:
+        - If token is length-2 and starts with a function char (在/对/从/与/和/将/为/把/被/给/向/于)
+          and the 2nd char is a plausible toponym initial (first char of any known ns entry),
+          split it into two single-char tokens.
+        - If token is length-2 and ends with an admin suffix (市/省/区/县/州/国/镇/村/旗/盟),
+          split it into (head char, suffix char).
+        - Then merge adjacent 2-char toponyms when they exist in the hand dictionary as ns.
+        """
+        if not toks:
+            return toks
+
+        func0 = {"在", "对", "从", "与", "和", "将", "为", "把", "被", "给", "向", "于"}
+        admin1 = {"市", "省", "县", "区", "州", "国", "镇", "村", "旗", "盟"}
+
+        # derive plausible toponym initials from known ns tokens
+        topo_initials = set()
+        try:
+            for w, p in self.dictionary.items():
+                if p == "ns" and isinstance(w, str) and len(w) >= 2:
+                    topo_initials.add(w[0])
+        except Exception:
+            topo_initials = set()
+
+        # step1: split tokens conservatively
+        split: List[Morpheme] = []
+        for m in toks:
+            s = m.surface
+            if isinstance(s, str) and len(s) == 2:
+                if (s[0] in func0) and (s[1] in topo_initials) and (s not in self.dictionary):
+                    # split: 在北 -> 在 + 北
+                    split.append(Morpheme(s[0], s[0], self.dictionary.get(s[0], "x"), m.start, m.start + 1))
+                    split.append(Morpheme(s[1], s[1], self.dictionary.get(s[1], "x"), m.start + 1, m.end))
+                    continue
+                if (s[1] in admin1) and (s not in self.dictionary):
+                    # split: 京市 -> 京 + 市
+                    split.append(Morpheme(s[0], s[0], self.dictionary.get(s[0], "x"), m.start, m.start + 1))
+                    split.append(Morpheme(s[1], s[1], self.dictionary.get(s[1], "x"), m.start + 1, m.end))
+                    continue
+            split.append(m)
+
+        # step2: merge adjacent 2-char toponyms when present in dictionary
+        out: List[Morpheme] = []
+        i = 0
+        while i < len(split):
+            if i + 1 < len(split):
+                a, b = split[i], split[i + 1]
+                if a.end == b.start:
+                    comb = a.surface + b.surface
+                    if self.dictionary.get(comb) == "ns":
+                        out.append(Morpheme(comb, comb, "ns", a.start, b.end))
+                        i += 2
+                        continue
+            out.append(split[i])
+            i += 1
+        return out
+
+    def _postprocess_merge_common_suffixes(self, toks: List[Morpheme]) -> List[Morpheme]:
+        """
+        Postprocess merges to improve segmentation quality for product use:
+        - Location suffixes: 北京 + 市 -> 北京市 (when previous token is a location-like ns)
+        - Organization suffixes: 阿里巴巴 + 集团 -> 阿里巴巴集团 (when previous is nrt)
+        - Numeric unit tails: 10 + 亿 -> 10亿 (handled in CJKTokenizer too, but keep safe here when we see it)
+
+        This is conservative: it only merges when contiguity is exact (prev.end == next.start),
+        and when previous token is already strongly typed (ns/nrt) or combined form exists in lexicon.
+        """
+        if not toks:
+            return toks
+
+        admin_suffix = {"市", "省", "县", "区", "州", "国", "镇", "村", "旗", "盟"}
+        # multi-char admin tails that frequently appear split
+        admin_suffix_multi = {"自治区", "自治州", "自治县", "特别行政区", "行政区"}
+        # NOTE: exclude "政府" here (it's a standalone noun too often; merging X+政府 is risky).
+        org_suffix_strong = {"集团", "公司", "银行", "大学"}
+        org_suffix_generic = {"委员会", "协会", "研究院", "研究所", "法院", "检察院", "公安局"}
+        # multi-char org tails
+        org_suffix_multi = {"有限公司", "有限责任公司", "股份有限公司", "集团公司"}
+
+        wf = self._wordfreq or {}
+
+        out: List[Morpheme] = []
+        i = 0
+        while i < len(toks):
+            cur = toks[i]
+            if out and cur.start == out[-1].end:
+                prev = out[-1]
+                comb = prev.surface + cur.surface
+                # 1) merge if combined form exists in hand dictionary
+                if comb in self.dictionary:
+                    pos = self.dictionary.get(comb, prev.pos)
+                    out[-1] = Morpheme(comb, comb, pos, prev.start, cur.end)
+                    i += 1
+                    continue
+                # 2) merge location + admin suffix (prev already ns)
+                if prev.pos == "ns" and cur.surface in admin_suffix:
+                    # Only merge if lexicon suggests it's a real unit OR prev is very short (typical toponyms)
+                    if wf.get(comb, 0) > 0 or len(prev.surface) <= 3:
+                        out[-1] = Morpheme(comb, comb, "ns", prev.start, cur.end)
+                        i += 1
+                        continue
+                # 2a) merge toponym + multi-char admin suffix (lexicon supported)
+                if cur.surface in admin_suffix_multi and 1 <= len(prev.surface) <= 6 and cur.start == prev.end:
+                    f_comb = int(wf.get(comb, 0) or 0)
+                    if prev.pos == "ns" or f_comb >= 200:
+                        out[-1] = Morpheme(comb, comb, "ns", prev.start, cur.end)
+                        i += 1
+                        continue
+                # 2b) merge toponym + admin suffix even if prev not tagged, when lexicon strongly supports comb.
+                if cur.surface in admin_suffix and 1 <= len(prev.surface) <= 4:
+                    f_comb = int(wf.get(comb, 0) or 0)
+                    if f_comb >= 100:
+                        out[-1] = Morpheme(comb, comb, "ns", prev.start, cur.end)
+                        i += 1
+                        continue
+                # 3) merge organization + suffix (prev already nrt)
+                if prev.pos == "nrt" and (cur.surface in org_suffix_strong or cur.surface in org_suffix_generic):
+                    # Prefer lexicon evidence, but be permissive for strong organization tails.
+                    if wf.get(comb, 0) > 0 or len(prev.surface) >= 2:
+                        out[-1] = Morpheme(comb, comb, "nrt", prev.start, cur.end)
+                        i += 1
+                        continue
+                # 3a) merge org + multi-char suffix (lexicon supported)
+                if cur.surface in org_suffix_multi and 1 <= len(prev.surface) <= 8 and cur.start == prev.end:
+                    f_comb = int(wf.get(comb, 0) or 0)
+                    if prev.pos == "nrt" or f_comb >= 200:
+                        out[-1] = Morpheme(comb, comb, "nrt", prev.start, cur.end)
+                        i += 1
+                        continue
+                # 3b) merge org tail even if prev not tagged, when lexicon supports comb.
+                # Avoid merging verb+bank (e.g., 支持 + 银行) via a tiny stoplist.
+                if cur.start == prev.end and 1 <= len(prev.surface) <= 8:
+                    stop_prev = {
+                        "支持", "提供", "表示", "认为", "指出", "强调", "包括", "进行", "开展", "推动", "加强",
+                        "将", "对", "在", "与", "和", "或", "但", "而", "为", "把", "被", "从", "向",
+                    }
+                    if prev.surface not in stop_prev:
+                        f_comb = int(wf.get(comb, 0) or 0)
+                        # strong tails: lower threshold
+                        if cur.surface in org_suffix_strong and f_comb >= 20:
+                            out[-1] = Morpheme(comb, comb, "nrt", prev.start, cur.end)
+                            i += 1
+                            continue
+                        # generic tails: keep strict
+                        if cur.surface in org_suffix_generic and f_comb >= 200:
+                            out[-1] = Morpheme(comb, comb, "nrt", prev.start, cur.end)
+                            i += 1
+                            continue
+            out.append(cur)
+            i += 1
+        return out
+
+    def _segment_with_lexicon(self, text: str) -> List[Morpheme]:
+        """
+        Segment full text. For Hanzi runs, use Viterbi over word candidates from:
+        - hand dictionary (high precision)
+        - wordfreq lexicon (coverage)
+        Non-Hanzi parts follow the same rules as the native segmenter.
+        """
+        out: List[Morpheme] = []
+        pos = 0
+        n = len(text)
+
+        while pos < n:
+            # optional date joins (reuse native date patterns)
+            if self.join_dates:
+                m = self._date_md_with_dian.match(text[pos:])
+                if m:
+                    s = m.group()
+                    core = s[:-1]
+                    out.append(Morpheme(core, core, 't', pos, pos + len(core)))
+                    out.append(Morpheme('电', '电', 'n', pos + len(core), pos + len(s)))
+                    pos += len(s)
+                    continue
+                m = self._date_ymd.match(text[pos:])
+                if m:
+                    s = m.group()
+                    out.append(Morpheme(s, s, 't', pos, pos + len(s)))
+                    pos += len(s)
+                    continue
+                m = self._date_md.match(text[pos:])
+                if m:
+                    s = m.group()
+                    out.append(Morpheme(s, s, 't', pos, pos + len(s)))
+                    pos += len(s)
+                    continue
+
+            ch = text[pos]
+            # whitespace
+            if ch.isspace():
+                pos += 1
+                continue
+
+            # latin/digit chunk
+            if not self.hanzi.match(ch):
+                match = re.match(r'[a-zA-Z0-9]+', text[pos:])
+                if match:
+                    w = match.group()
+                    out.append(Morpheme(w, w, 'x', pos, pos + len(w)))
+                    pos += len(w)
+                else:
+                    out.append(Morpheme(ch, ch, 'x', pos, pos + 1))
+                    pos += 1
+                continue
+
+            # hanzi run
+            m = self.hanzi.match(text[pos:])
+            if not m:
+                out.append(Morpheme(ch, ch, 'x', pos, pos + 1))
+                pos += 1
+                continue
+            run = m.group()
+            run_start = pos
+            out.extend(self._viterbi_hanzi_run(run, run_start))
+            pos += len(run)
+
+        return out
+
+    def _viterbi_hanzi_run(self, run: str, offset: int) -> List[Morpheme]:
+        """
+        Viterbi segmentation over a pure-Hanzi run.
+        Score = log(freq+1) + len_bonus*(len-1) - single_penalty(if len==1) + dict_bonus(if in hand dict)
+        """
+        wf = self._wordfreq or {}
+        max_len = max(self.max_word_len, self._wordfreq_max_len)
+        max_len = max(2, min(int(max_len), 8))
+        n = len(run)
+
+        len_bonus = 0.8
+        single_penalty = 1.2
+        # Hand dictionary entries should beat high-frequency short n-grams.
+        # Otherwise entities like "阿里巴巴" can get split into "阿里"+"巴巴".
+        dict_bonus = 3.5
+        dict_len_bonus = 0.6
+        entity_bonus = 4.0
+        entity_freq_floor = 50_000
+        # When lexicon doesn't contain a span, still allow 2-4 char grouping
+        # to avoid degenerate per-character segmentation.
+        unk_base = -1.5
+        unk_len_penalty = 0.35
+        freq_cap = 200_000
+
+        best_score = [-1e100] * (n + 1)
+        back = [-1] * (n + 1)
+        back_len = [1] * (n + 1)
+        best_score[0] = 0.0
+
+        for i in range(n):
+            if best_score[i] <= -1e90:
+                continue
+            # try candidates
+            for L in range(1, min(max_len, n - i) + 1):
+                w = run[i:i+L]
+                # require either hand dict or wordfreq for multi-char; allow single-char always
+                freq = wf.get(w, 0)
+                in_dict = w in self.dictionary
+                # If it's a known entity/location in the hand dictionary, treat it as high-confidence.
+                # This prevents frequent short n-grams from splitting proper nouns (e.g., 阿里巴巴 -> 阿里 + 巴巴).
+                if in_dict:
+                    pos_tag = self.dictionary.get(w, "")
+                    if pos_tag in ("nrt", "ns"):
+                        freq = max(freq, entity_freq_floor)
+                sc = best_score[i]
+                if in_dict or freq > 0:
+                    # base by frequency
+                    sc += math.log(min(freq, freq_cap) + 1.0)
+                else:
+                    # unknown grouping: prefer 2-3 char chunks over 1-char
+                    sc += unk_base - unk_len_penalty * L
+                # length preference (avoid too many singles)
+                sc += len_bonus * (L - 1)
+                if L == 1:
+                    sc -= single_penalty
+                if in_dict:
+                    sc += dict_bonus + dict_len_bonus * (L - 1)
+                    pos_tag = self.dictionary.get(w, "")
+                    if pos_tag in ("nrt", "ns"):
+                        sc += entity_bonus
+                j = i + L
+                if sc > best_score[j]:
+                    best_score[j] = sc
+                    back[j] = i
+                    back_len[j] = L
+
+        # if something went wrong, fallback to single chars
+        if back[n] < 0:
+            return [Morpheme(run[i], run[i], self.dictionary.get(run[i], "n"), offset + i, offset + i + 1) for i in range(n)]
+
+        # backtrack
+        toks: List[Tuple[int, int]] = []
+        j = n
+        while j > 0:
+            i = back[j]
+            if i < 0:
+                # safety
+                i = j - 1
+            L = back_len[j]
+            toks.append((i, j))
+            j = i
+        toks.reverse()
+
+        out: List[Morpheme] = []
+        for i, j in toks:
+            w = run[i:j]
+            pos = self.dictionary.get(w, "x")
+            out.append(Morpheme(w, w, pos, offset + i, offset + j))
+        return out
+
+    def _forward_max_match(self, text: str) -> List[Morpheme]:
+        """정방향 최장일치"""
+        result = []
+        pos = 0
+
+        while pos < len(text):
+            # 날짜 패턴 우선 처리 (숫자+年/月/日/号) - optional
+            if self.join_dates:
+                m = self._date_md_with_dian.match(text[pos:])
+                if m:
+                    s = m.group()
+                    # split: 12月31日 + 电
+                    core = s[:-1]
+                    result.append(Morpheme(core, core, 't', pos, pos + len(core)))
+                    result.append(Morpheme('电', '电', 'n', pos + len(core), pos + len(s)))
+                    pos += len(s)
+                    continue
+                m = self._date_ymd.match(text[pos:])
+                if m:
+                    s = m.group()
+                    result.append(Morpheme(s, s, 't', pos, pos + len(s)))
+                    pos += len(s)
+                    continue
+                m = self._date_md.match(text[pos:])
+                if m:
+                    s = m.group()
+                    result.append(Morpheme(s, s, 't', pos, pos + len(s)))
+                    pos += len(s)
+                    continue
+
+            # 공백/기호 스킵
+            if not self.hanzi.match(text[pos:pos+1]):
+                if text[pos].isspace():
+                    pos += 1
+                    continue
+                # 숫자/영문
+                match = re.match(r'[a-zA-Z0-9]+', text[pos:])
+                if match:
+                    word = match.group()
+                    result.append(Morpheme(word, word, 'x', pos, pos + len(word)))
+                    pos += len(word)
+                else:
+                    result.append(Morpheme(text[pos], text[pos], 'x', pos, pos + 1))
+                    pos += 1
+                continue
+
+            # 최장일치
+            matched = False
+            for length in range(min(self.max_word_len, len(text) - pos), 0, -1):
+                word = text[pos:pos+length]
+                if word in self.dictionary:
+                    result.append(Morpheme(
+                        word, word, self.dictionary[word],
+                        pos, pos + length
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+            if not matched:
+                # 미등록어: 다음 사전 단어까지 그룹화 (2-4자 우선)
+                end_pos = pos + 1
+                while end_pos < len(text) and end_pos - pos < 4:
+                    if not self.hanzi.match(text[end_pos:end_pos+1]):
+                        break
+                    # 이 위치에서 사전 단어가 시작하면 끊음
+                    found_dict_word = False
+                    for length in range(min(self.max_word_len, len(text) - end_pos), 0, -1):
+                        if text[end_pos:end_pos+length] in self.dictionary:
+                            found_dict_word = True
+                            break
+                    if found_dict_word:
+                        break
+                    end_pos += 1
+
+                # 2자 이상이면 하나의 단어로
+                if end_pos - pos >= 2:
+                    word = text[pos:end_pos]
+                    result.append(Morpheme(
+                        word, word, 'nz',  # nz = 미등록 고유명사
+                        pos, end_pos
+                    ))
+                    pos = end_pos
+                else:
+                    # 1자는 그대로
+                    result.append(Morpheme(
+                        text[pos], text[pos], 'n',
+                        pos, pos + 1
+                    ))
+                    pos += 1
+
+        return result
+
+    def _backward_max_match(self, text: str) -> List[Morpheme]:
+        """역방향 최장일치"""
+        result = []
+        pos = len(text)
+
+        while pos > 0:
+            # 공백 스킵
+            if pos > 0 and text[pos-1].isspace():
+                pos -= 1
+                continue
+
+            # 비한자
+            if pos > 0 and not self.hanzi.match(text[pos-1:pos]):
+                # 연속 비한자 찾기
+                end = pos
+                while pos > 0 and not self.hanzi.match(text[pos-1:pos]) and not text[pos-1].isspace():
+                    pos -= 1
+                if pos < end:
+                    word = text[pos:end]
+                    result.insert(0, Morpheme(word, word, 'x', pos, end))
+                continue
+
+            # 날짜 패턴(역방향) 처리(옵션): "...12月31日电" / "...2025年12月31日"
+            if self.join_dates:
+                lookback = max(0, pos - 16)
+                chunk = text[lookback:pos]
+                m = re.search(r'[0-9]{1,2}月[0-9]{1,2}[日号]电$', chunk)
+                if m:
+                    s = m.group()
+                    core = s[:-1]
+                    start = pos - len(s)
+                    result.insert(0, Morpheme('电', '电', 'n', pos - 1, pos))
+                    result.insert(0, Morpheme(core, core, 't', start, start + len(core)))
+                    pos -= len(s)
+                    continue
+                m = re.search(r'[0-9]{2,4}年[0-9]{1,2}月[0-9]{1,2}[日号]$', chunk)
+                if m:
+                    s = m.group()
+                    start = pos - len(s)
+                    result.insert(0, Morpheme(s, s, 't', start, pos))
+                    pos -= len(s)
+                    continue
+                m = re.search(r'[0-9]{1,2}月[0-9]{1,2}[日号]$', chunk)
+                if m:
+                    s = m.group()
+                    start = pos - len(s)
+                    result.insert(0, Morpheme(s, s, 't', start, pos))
+                    pos -= len(s)
+                    continue
+
+            # 최장일치 (역방향)
+            matched = False
+            for length in range(min(self.max_word_len, pos), 0, -1):
+                word = text[pos-length:pos]
+                if word in self.dictionary:
+                    result.insert(0, Morpheme(
+                        word, word, self.dictionary[word],
+                        pos - length, pos
+                    ))
+                    pos -= length
+                    matched = True
+                    break
+
+            if not matched:
+                # 미등록어: 이전 사전 단어까지 그룹화 (역방향, 2-4자)
+                start_pos = pos - 1
+                while start_pos > 0 and pos - start_pos < 4:
+                    if not self.hanzi.match(text[start_pos-1:start_pos]):
+                        break
+                    # 이 위치에서 끝나는 사전 단어가 있으면 끊음
+                    found_dict_word = False
+                    for length in range(min(self.max_word_len, start_pos), 0, -1):
+                        if text[start_pos-length:start_pos] in self.dictionary:
+                            found_dict_word = True
+                            break
+                    if found_dict_word:
+                        break
+                    start_pos -= 1
+
+                # 2자 이상이면 하나의 단어로
+                if pos - start_pos >= 2:
+                    word = text[start_pos:pos]
+                    result.insert(0, Morpheme(
+                        word, word, 'nz',  # nz = 미등록 고유명사
+                        start_pos, pos
+                    ))
+                    pos = start_pos
+                else:
+                    result.insert(0, Morpheme(
+                        text[pos-1], text[pos-1], 'n',
+                        pos - 1, pos
+                    ))
+                    pos -= 1
+
+        return result
+
+    def segment(self, text: str) -> List[str]:
+        """단어 분리 (간편 함수)"""
+        morphemes = self.analyze(text)
+        return [m.surface for m in morphemes]
+
+    def pos_tag(self, text: str) -> List[Tuple[str, str]]:
+        """품사 태깅"""
+        morphemes = self.analyze(text)
+        return [(m.surface, m.pos) for m in morphemes]