tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/chinese_advanced.py

@@ -0,0 +1,425 @@
+"""
+Chinese Advanced Morphological Analyzer
+=======================================
+
+5가지 고급 기능을 지원하는 중국어 형태소 분석기
+
+Features:
+1. NER Gazetteer Integration - 개체명 경계 보존
+2. Real-time Dictionary Extension - 런타임 사전 확장
+3. Domain Adaptation - 도메인별 분석 최적화
+4. Code-switching - 영중 혼용 텍스트 처리
+5. N-best Analysis - 다중 후보 + 신뢰도 점수
+
+Algorithm: Bidirectional Maximum Matching
+"""
+
+import re
+from typing import List, Tuple, Dict, Set, Optional, Any
+
+from .advanced_base import (
+    AdvancedMorphologicalAnalyzer, Morpheme, AnalysisResult, NBestResult, Domain
+)
+
+
+class ChineseAdvancedAnalyzer(AdvancedMorphologicalAnalyzer):
+    """
+    중국어 고급 형태소 분석기
+
+    Usage:
+        analyzer = ChineseAdvancedAnalyzer()
+
+        # 기본 분석
+        result = analyzer.analyze("阿里巴巴集团在杭州宣布")
+
+        # 개체명 보존
+        analyzer.add_entity("阿里巴巴", "ORG")
+        result = analyzer.analyze("阿里巴巴在北京", preserve_entities=True)
+
+        # 도메인 적응
+        result = analyzer.analyze("苹果很好吃", domain="food")  # 사과
+        result = analyzer.analyze("苹果发布新品", domain="tech")  # Apple Inc.
+
+        # N-best 분석
+        result = analyzer.analyze("银行", n_best=3)
+    """
+
+    LANG_CODE = "zh"
+    LANG_NAME = "Chinese"
+
+    # 한자 패턴
+    HANZI_PATTERN = re.compile(r'[\u4e00-\u9fff]+')
+    LATIN_PATTERN = re.compile(r'[a-zA-Z]+')
+    NUMBER_PATTERN = re.compile(r'[0-9]+')
+
+    def __init__(self):
+        super().__init__()
+        self.max_word_len = 8  # 최대 단어 길이
+
+    def _build_base_dictionary(self):
+        """기본 사전 구축"""
+
+        # =================================================================
+        # 地名 (지명)
+        # =================================================================
+        self.places = {
+            '北京': 'ns', '上海': 'ns', '广州': 'ns', '深圳': 'ns',
+            '杭州': 'ns', '南京': 'ns', '武汉': 'ns', '成都': 'ns',
+            '西安': 'ns', '重庆': 'ns', '天津': 'ns', '苏州': 'ns',
+            '香港': 'ns', '台湾': 'ns', '澳门': 'ns',
+            '中国': 'ns', '美国': 'ns', '日本': 'ns', '韩国': 'ns',
+            '英国': 'ns', '法国': 'ns', '德国': 'ns', '俄罗斯': 'ns',
+        }
+
+        # =================================================================
+        # 组织/企业 (조직/기업)
+        # =================================================================
+        self.organizations = {
+            '阿里巴巴': 'nrt', '腾讯': 'nrt', '百度': 'nrt', '华为': 'nrt',
+            '小米': 'nrt', '京东': 'nrt', '美团': 'nrt', '字节跳动': 'nrt',
+            '苹果': 'nrt',  # Apple (tech context)
+            '三星': 'nrt', '谷歌': 'nrt', '微软': 'nrt', '亚马逊': 'nrt',
+        }
+
+        # =================================================================
+        # 一般名词 (일반명사)
+        # =================================================================
+        self.common_nouns = {
+            # 组织
+            '集团': 'n', '公司': 'n', '企业': 'n', '银行': 'n',
+            '政府': 'n', '学校': 'n', '大学': 'n', '医院': 'n',
+            # 一般
+            '人': 'n', '人们': 'n', '时间': 'n', '地方': 'n',
+            '问题': 'n', '情况': 'n', '工作': 'n', '生活': 'n',
+            '发展': 'n', '经济': 'n', '社会': 'n', '文化': 'n',
+            '技术': 'n', '产品': 'n', '服务': 'n', '市场': 'n',
+            '国家': 'n', '世界': 'n', '城市': 'n', '地区': 'n',
+            '消息': 'n', '新闻': 'n', '报道': 'n', '信息': 'n',
+            # 食物 (음식)
+            '苹果': 'n',  # 사과 (food context)
+            '香蕉': 'n', '橘子': 'n', '西瓜': 'n',
+            '米饭': 'n', '面条': 'n', '饺子': 'n',
+        }
+
+        # =================================================================
+        # 动词 (동사)
+        # =================================================================
+        self.verbs = {
+            '是': 'v', '有': 'v', '在': 'v', '说': 'v', '做': 'v',
+            '去': 'v', '来': 'v', '看': 'v', '想': 'v', '知道': 'v',
+            '发表': 'v', '宣布': 'v', '公布': 'v', '发布': 'v',
+            '开始': 'v', '结束': 'v', '进行': 'v', '完成': 'v',
+            '研究': 'v', '开发': 'v', '生产': 'v', '销售': 'v',
+            '投资': 'v', '合作': 'v', '成立': 'v', '成为': 'v',
+            '表示': 'v', '认为': 'v', '希望': 'v', '需要': 'v',
+            '吃': 'v', '喝': 'v', '买': 'v', '卖': 'v',
+        }
+
+        # =================================================================
+        # 形容词 (형용사)
+        # =================================================================
+        self.adjectives = {
+            '大': 'a', '小': 'a', '多': 'a', '少': 'a',
+            '好': 'a', '新': 'a', '高': 'a', '低': 'a',
+            '重要': 'a', '主要': 'a', '不同': 'a', '相同': 'a',
+        }
+
+        # =================================================================
+        # 副词 (부사)
+        # =================================================================
+        self.adverbs = {
+            '不': 'd', '也': 'd', '就': 'd', '都': 'd', '还': 'd',
+            '很': 'd', '最': 'd', '已经': 'd', '正在': 'd',
+            '可能': 'd', '一定': 'd', '非常': 'd', '比较': 'd',
+        }
+
+        # =================================================================
+        # 介词 (개사/전치사)
+        # =================================================================
+        self.prepositions = {
+            '在': 'p', '从': 'p', '向': 'p', '对': 'p', '把': 'p',
+            '被': 'p', '比': 'p', '跟': 'p', '和': 'p', '与': 'p',
+        }
+
+        # =================================================================
+        # 助词 (조사)
+        # =================================================================
+        self.particles = {
+            '的': 'u', '地': 'u', '得': 'u', '了': 'u', '着': 'u', '过': 'u',
+            '吗': 'u', '呢': 'u', '吧': 'u', '啊': 'u',
+        }
+
+        # =================================================================
+        # 代词 (대명사)
+        # =================================================================
+        self.pronouns = {
+            '我': 'r', '你': 'r', '他': 'r', '她': 'r', '它': 'r',
+            '我们': 'r', '你们': 'r', '他们': 'r',
+            '这': 'r', '那': 'r', '这个': 'r', '那个': 'r',
+            '什么': 'r', '谁': 'r', '哪': 'r', '怎么': 'r',
+        }
+
+        # =================================================================
+        # 连词 (연결사)
+        # =================================================================
+        self.conjunctions = {
+            '和': 'c', '或': 'c', '但': 'c', '但是': 'c',
+            '因为': 'c', '所以': 'c', '如果': 'c', '虽然': 'c',
+        }
+
+        # =================================================================
+        # 数词 (수사)
+        # =================================================================
+        self.numerals = {
+            '一': 'm', '二': 'm', '三': 'm', '四': 'm', '五': 'm',
+            '六': 'm', '七': 'm', '八': 'm', '九': 'm', '十': 'm',
+            '百': 'm', '千': 'm', '万': 'm', '亿': 'm',
+            '两': 'm', '几': 'm', '多': 'm',
+        }
+
+        # =================================================================
+        # 量词 (양사)
+        # =================================================================
+        self.classifiers = {
+            '个': 'q', '年': 'q', '月': 'q', '日': 'q', '号': 'q',
+            '次': 'q', '种': 'q', '件': 'q', '位': 'q', '家': 'q',
+        }
+
+        # 통합 사전 구축
+        self._dictionary = {}
+        for d in [self.places, self.organizations, self.common_nouns,
+                  self.verbs, self.adjectives, self.adverbs,
+                  self.prepositions, self.particles, self.pronouns,
+                  self.conjunctions, self.numerals, self.classifiers]:
+            for word, pos in d.items():
+                self._dictionary[word] = (word, pos)
+
+    def _build_domain_dictionaries(self):
+        """도메인별 사전 구축"""
+
+        # FOOD 도메인
+        self._domain_dictionaries[Domain.FOOD] = {
+            '苹果': ('苹果', 'n'),  # 사과
+            '小米': ('小米', 'n'),  # 좁쌀
+            '银行': ('银行', 'n'),  # 일반 은행
+        }
+
+        # TECH 도메인
+        self._domain_dictionaries[Domain.TECH] = {
+            '苹果': ('苹果公司', 'nrt'),  # Apple Inc.
+            '小米': ('小米科技', 'nrt'),  # Xiaomi
+            '华为': ('华为技术', 'nrt'),
+            '云': ('云计算', 'n'),
+        }
+
+        # FINANCE 도메인
+        self._domain_dictionaries[Domain.FINANCE] = {
+            '银行': ('银行', 'n'),
+            '股票': ('股票', 'n'),
+            '基金': ('基金', 'n'),
+            '投资': ('投资', 'n'),
+        }
+
+        # ENTERTAINMENT 도메인
+        self._domain_dictionaries[Domain.ENTERTAINMENT] = {
+            '苹果': ('苹果', 'n'),  # 일반 사과
+            '明星': ('明星', 'n'),
+            '演员': ('演员', 'n'),
+        }
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        """분석 후보 생성 (양방향 최장일치)"""
+        if not text or not text.strip():
+            return [AnalysisResult([])]
+
+        candidates = []
+
+        # 정방향 분석
+        forward_morphemes = self._forward_max_match(text, domain)
+        forward_result = AnalysisResult(
+            morphemes=forward_morphemes,
+            score=1.0,
+            domain=domain
+        )
+
+        # 역방향 분석
+        backward_morphemes = self._backward_max_match(text, domain)
+        backward_result = AnalysisResult(
+            morphemes=backward_morphemes,
+            score=0.95,
+            domain=domain
+        )
+
+        # 더 적은 형태소 수를 선택 (기본 휴리스틱)
+        if len(forward_morphemes) <= len(backward_morphemes):
+            forward_result.score = self._score_analysis(forward_result)
+            candidates.append(forward_result)
+            backward_result.score = self._score_analysis(backward_result) * 0.9
+            candidates.append(backward_result)
+        else:
+            backward_result.score = self._score_analysis(backward_result)
+            candidates.append(backward_result)
+            forward_result.score = self._score_analysis(forward_result) * 0.9
+            candidates.append(forward_result)
+
+        return candidates
+
+    def _forward_max_match(self, text: str, domain: Domain) -> List[Morpheme]:
+        """정방향 최장일치"""
+        result = []
+        pos = 0
+
+        while pos < len(text):
+            # 공백 스킵
+            if text[pos].isspace():
+                pos += 1
+                continue
+
+            # 비한자 처리
+            if not self.HANZI_PATTERN.match(text[pos:pos+1]):
+                # 라틴 문자
+                match = self.LATIN_PATTERN.match(text[pos:])
+                if match:
+                    word = match.group()
+                    # 런타임 사전 확인
+                    if word.lower() in self._user_dictionary:
+                        lemma, pos_tag, _ = self._user_dictionary[word.lower()]
+                        result.append(Morpheme(word, lemma, pos_tag, pos, pos + len(word)))
+                    else:
+                        result.append(Morpheme(word, word, 'x', pos, pos + len(word)))
+                    pos += len(word)
+                    continue
+
+                # 숫자
+                match = self.NUMBER_PATTERN.match(text[pos:])
+                if match:
+                    word = match.group()
+                    result.append(Morpheme(word, word, 'm', pos, pos + len(word)))
+                    pos += len(word)
+                    continue
+
+                # 기타 기호
+                result.append(Morpheme(text[pos], text[pos], 'x', pos, pos + 1))
+                pos += 1
+                continue
+
+            # 최장일치 (런타임 사전 우선)
+            matched = False
+
+            # 런타임 사전
+            for length in range(min(self.max_word_len, len(text) - pos), 0, -1):
+                word = text[pos:pos+length]
+                if word in self._user_dictionary:
+                    lemma, pos_tag, _ = self._user_dictionary[word]
+                    result.append(Morpheme(word, lemma, pos_tag, pos, pos + length))
+                    pos += length
+                    matched = True
+                    break
+
+            if matched:
+                continue
+
+            # 도메인 사전
+            domain_sense = None
+            for length in range(min(self.max_word_len, len(text) - pos), 0, -1):
+                word = text[pos:pos+length]
+                domain_sense = self._get_domain_sense(word, domain)
+                if domain_sense:
+                    result.append(Morpheme(word, domain_sense[0], domain_sense[1], pos, pos + length))
+                    pos += length
+                    matched = True
+                    break
+
+            if matched:
+                continue
+
+            # 기본 사전
+            for length in range(min(self.max_word_len, len(text) - pos), 0, -1):
+                word = text[pos:pos+length]
+                if word in self._dictionary:
+                    lemma, pos_tag = self._dictionary[word]
+                    result.append(Morpheme(word, lemma, pos_tag, pos, pos + length))
+                    pos += length
+                    matched = True
+                    break
+
+            if not matched:
+                # 미등록어: 한 글자씩
+                result.append(Morpheme(text[pos], text[pos], 'n', pos, pos + 1))
+                pos += 1
+
+        return result
+
+    def _backward_max_match(self, text: str, domain: Domain) -> List[Morpheme]:
+        """역방향 최장일치"""
+        result = []
+        pos = len(text)
+
+        while pos > 0:
+            # 공백 스킵
+            if text[pos-1].isspace():
+                pos -= 1
+                continue
+
+            # 비한자
+            if not self.HANZI_PATTERN.match(text[pos-1:pos]):
+                end = pos
+                while pos > 0 and not self.HANZI_PATTERN.match(text[pos-1:pos]) and not text[pos-1].isspace():
+                    pos -= 1
+                if pos < end:
+                    word = text[pos:end]
+                    result.insert(0, Morpheme(word, word, 'x', pos, end))
+                continue
+
+            # 최장일치 (역방향)
+            matched = False
+
+            # 런타임 사전
+            for length in range(min(self.max_word_len, pos), 0, -1):
+                word = text[pos-length:pos]
+                if word in self._user_dictionary:
+                    lemma, pos_tag, _ = self._user_dictionary[word]
+                    result.insert(0, Morpheme(word, lemma, pos_tag, pos - length, pos))
+                    pos -= length
+                    matched = True
+                    break
+
+            if matched:
+                continue
+
+            # 기본 사전
+            for length in range(min(self.max_word_len, pos), 0, -1):
+                word = text[pos-length:pos]
+                if word in self._dictionary:
+                    lemma, pos_tag = self._dictionary[word]
+                    result.insert(0, Morpheme(word, lemma, pos_tag, pos - length, pos))
+                    pos -= length
+                    matched = True
+                    break
+
+            if not matched:
+                result.insert(0, Morpheme(text[pos-1], text[pos-1], 'n', pos - 1, pos))
+                pos -= 1
+
+        return result
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        """대안 분석 결과 생성"""
+        # 이미 양방향 분석에서 2개 후보 생성
+        alternatives = []
+
+        # 다른 도메인으로 분석
+        other_domains = [d for d in Domain if d != domain][:count]
+
+        for alt_domain in other_domains:
+            candidates = self._generate_candidates(text, alt_domain)
+            for c in candidates[:1]:
+                c.score *= 0.85
+                c.domain = alt_domain
+                alternatives.append(c)
+
+        return alternatives
+
+
+# Alias for backward compatibility
+ChineseAnalyzer = ChineseAdvancedAnalyzer