tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/japanese_advanced.py

@@ -0,0 +1,589 @@
+"""
+Japanese Advanced Morphological Analyzer
+========================================
+
+5가지 고급 기능을 지원하는 일본어 형태소 분석기
+
+Features:
+1. NER Gazetteer Integration - 개체명 경계 보존
+2. Real-time Dictionary Extension - 런타임 사전 확장
+3. Domain Adaptation - 도메인별 분석 최적화
+4. Code-switching - 영일 혼용 텍스트 처리
+5. N-best Analysis - 다중 후보 + 신뢰도 점수
+"""
+
+import re
+import json
+from pathlib import Path
+from typing import List, Tuple, Dict, Set, Optional, Any
+
+from .advanced_base import (
+    AdvancedMorphologicalAnalyzer, Morpheme, AnalysisResult, NBestResult, Domain
+)
+
+# 확장 사전 경로
+from .. import resources
+
+# Optional external asset dir (default: none). If you want extended dictionaries,
+# provide them under: TOKMOR_DATA_DIR/extended_dict/{lang}_extended.json
+DICT_DIR = resources.data_dir() / "extended_dict"
+
+
+class JapaneseAdvancedAnalyzer(AdvancedMorphologicalAnalyzer):
+    """
+    일본어 고급 형태소 분석기
+
+    Usage:
+        analyzer = JapaneseAdvancedAnalyzer()
+
+        # 기본 분석
+        result = analyzer.analyze("東京に行きます")
+
+        # 개체명 보존
+        analyzer.add_entity("東京大学", "ORG")
+        result = analyzer.analyze("東京大学に行きます", preserve_entities=True)
+
+        # 도메인 적응
+        result = analyzer.analyze("株を買う", domain="finance")
+
+        # N-best 분석
+        result = analyzer.analyze("橋", n_best=3)
+    """
+
+    LANG_CODE = "ja"
+    LANG_NAME = "Japanese"
+
+    # Unicode patterns
+    HIRAGANA_PATTERN = re.compile(r'[\u3040-\u309f]+')
+    KATAKANA_PATTERN = re.compile(r'[\u30a0-\u30ff]+')
+    KANJI_PATTERN = re.compile(r'[\u4e00-\u9fff]+')
+    LATIN_PATTERN = re.compile(r'[a-zA-Z]+')
+    NUMBER_PATTERN = re.compile(r'[0-9]+')
+
+    def __init__(self):
+        super().__init__()
+
+    def _build_base_dictionary(self):
+        """기본 사전 구축"""
+
+        # =================================================================
+        # 助詞 (조사)
+        # =================================================================
+        self.particles = {
+            # 格助詞
+            'は': 'HA', 'が': 'GA', 'を': 'WO', 'に': 'NI', 'へ': 'HE',
+            'で': 'DE', 'と': 'TO', 'から': 'KARA', 'まで': 'MADE',
+            'より': 'YORI', 'の': 'NO',
+            # 接続助詞
+            'ば': 'BA', 'たら': 'TARA', 'ても': 'TEMO', 'けど': 'KEDO',
+            'けれど': 'KEREDO', 'が': 'GA', 'のに': 'NONI',
+            # 副助詞
+            'も': 'MO', 'だけ': 'DAKE', 'しか': 'SHIKA', 'ばかり': 'BAKARI',
+            'など': 'NADO', 'くらい': 'KURAI', 'ほど': 'HODO',
+            # 終助詞
+            'か': 'KA', 'ね': 'NE', 'よ': 'YO', 'な': 'NA', 'わ': 'WA',
+            'ぞ': 'ZO', 'さ': 'SA',
+        }
+
+        # =================================================================
+        # 助動詞 (조동사)
+        # =================================================================
+        self.auxiliaries = {
+            'です': 'AUX', 'ます': 'AUX', 'た': 'AUX', 'だ': 'AUX',
+            'ない': 'AUX', 'れる': 'AUX', 'られる': 'AUX',
+            'せる': 'AUX', 'させる': 'AUX', 'たい': 'AUX',
+            'ている': 'AUX', 'てる': 'AUX', 'ました': 'AUX',
+            'でした': 'AUX', 'ません': 'AUX', 'ではない': 'AUX',
+        }
+
+        # =================================================================
+        # 名詞 (명사)
+        # =================================================================
+        self.nouns = {
+            # 地名
+            '東京': ('名詞', 'トウキョウ'),
+            '大阪': ('名詞', 'オオサカ'),
+            '京都': ('名詞', 'キョウト'),
+            '日本': ('名詞', 'ニホン'),
+            '中国': ('名詞', 'チュウゴク'),
+            '韓国': ('名詞', 'カンコク'),
+            'アメリカ': ('名詞', 'アメリカ'),
+            # 組織
+            '会社': ('名詞', 'カイシャ'),
+            '学校': ('名詞', 'ガッコウ'),
+            '大学': ('名詞', 'ダイガク'),
+            '政府': ('名詞', 'セイフ'),
+            '銀行': ('名詞', 'ギンコウ'),
+            # 一般名詞
+            '人': ('名詞', 'ヒト'),
+            '仕事': ('名詞', 'シゴト'),
+            '時間': ('名詞', 'ジカン'),
+            '今日': ('名詞', 'キョウ'),
+            '明日': ('名詞', 'アシタ'),
+            '昨日': ('名詞', 'キノウ'),
+            '発表': ('名詞', 'ハッピョウ'),
+            '自動車': ('名詞', 'ジドウシャ'),
+            '電話': ('名詞', 'デンワ'),
+            '食べ物': ('名詞', 'タベモノ'),
+            '飲み物': ('名詞', 'ノミモノ'),
+            '橋': ('名詞', 'ハシ'),  # 다의어: 橋/箸/端
+            '株': ('名詞', 'カブ'),  # 다의어: 株式/木の株
+            # 代名詞
+            '私': ('代名詞', 'ワタシ'),
+            '僕': ('代名詞', 'ボク'),
+            '彼': ('代名詞', 'カレ'),
+            '彼女': ('代名詞', 'カノジョ'),
+            'これ': ('代名詞', 'コレ'),
+            'それ': ('代名詞', 'ソレ'),
+            'あれ': ('代名詞', 'アレ'),
+        }
+
+        # =================================================================
+        # 動詞 (동사) - 어간 + 활용 타입
+        # =================================================================
+        self.verbs = {
+            # 5段動詞
+            '行': ('動詞', '行く', 'godan_k'),
+            '書': ('動詞', '書く', 'godan_k'),
+            '聞': ('動詞', '聞く', 'godan_k'),
+            '読': ('動詞', '読む', 'godan_m'),
+            '飲': ('動詞', '飲む', 'godan_m'),
+            '話': ('動詞', '話す', 'godan_s'),
+            '待': ('動詞', '待つ', 'godan_t'),
+            '買': ('動詞', '買う', 'godan_w'),
+            '言': ('動詞', '言う', 'godan_w'),
+            '思': ('動詞', '思う', 'godan_w'),
+            # 1段動詞
+            '見': ('動詞', '見る', 'ichidan'),
+            '食': ('動詞', '食べる', 'ichidan'),
+            '起': ('動詞', '起きる', 'ichidan'),
+            '寝': ('動詞', '寝る', 'ichidan'),
+            # カ変・サ変
+            '来': ('動詞', '来る', 'kuru'),
+            'し': ('動詞', 'する', 'suru'),
+            'する': ('動詞', 'する', 'suru'),
+        }
+
+        # =================================================================
+        # 形容詞 (형용사) - 기본형 + 어간형
+        # =================================================================
+        self.adjectives = {
+            # 기본형 (終止形)
+            '大きい': ('形容詞', '大きい'),
+            '小さい': ('形容詞', '小さい'),
+            '高い': ('形容詞', '高い'),
+            '安い': ('形容詞', '安い'),
+            '新しい': ('形容詞', '新しい'),
+            '古い': ('形容詞', '古い'),
+            '良い': ('形容詞', '良い'),
+            'いい': ('形容詞', '良い'),  # 口語形
+            '悪い': ('形容詞', '悪い'),
+            '美しい': ('形容詞', '美しい'),
+            '嬉しい': ('形容詞', '嬉しい'),
+            '楽しい': ('形容詞', '楽しい'),
+            '難しい': ('形容詞', '難しい'),
+            '早い': ('形容詞', '早い'),
+            '速い': ('形容詞', '速い'),
+            '遅い': ('形容詞', '遅い'),
+            '若い': ('形容詞', '若い'),
+            '白い': ('形容詞', '白い'),
+            '黒い': ('形容詞', '黒い'),
+            '赤い': ('形容詞', '赤い'),
+            '青い': ('形容詞', '青い'),
+            '長い': ('形容詞', '長い'),
+            '短い': ('形容詞', '短い'),
+            '多い': ('形容詞', '多い'),
+            '少ない': ('形容詞', '少ない'),
+            '強い': ('形容詞', '強い'),
+            '弱い': ('形容詞', '弱い'),
+            '広い': ('形容詞', '広い'),
+            '狭い': ('形容詞', '狭い'),
+            '近い': ('形容詞', '近い'),
+            '遠い': ('形容詞', '遠い'),
+            # 어간 (活用形에서 사용)
+            '大き': ('形容詞', '大きい'),
+            '小さ': ('形容詞', '小さい'),
+            '高': ('形容詞', '高い'),
+            '安': ('形容詞', '安い'),
+            '新し': ('形容詞', '新しい'),
+            '古': ('形容詞', '古い'),
+            '良': ('形容詞', '良い'),
+            '悪': ('形容詞', '悪い'),
+            '美し': ('形容詞', '美しい'),
+            '嬉し': ('形容詞', '嬉しい'),
+        }
+
+        # =================================================================
+        # 動詞活用形 (동사 활용형)
+        # =================================================================
+        self.verb_forms = {
+            # 行く
+            '行きます': [('行', '動詞'), ('き', '連用形'), ('ます', '助動詞')],
+            '行った': [('行', '動詞'), ('っ', '促音便'), ('た', '助動詞')],
+            '行く': [('行', '動詞'), ('く', '終止形')],
+            '行かない': [('行', '動詞'), ('か', '未然形'), ('ない', '助動詞')],
+            # 来る
+            '来ます': [('来', '動詞'), ('ます', '助動詞')],
+            '来た': [('来', '動詞'), ('た', '助動詞')],
+            '来ない': [('来', '動詞'), ('ない', '助動詞')],
+            # する
+            'します': [('し', '動詞'), ('ます', '助動詞')],
+            'した': [('し', '動詞'), ('た', '助動詞')],
+            'しない': [('し', '動詞'), ('ない', '助動詞')],
+            # 食べる
+            '食べます': [('食', '動詞'), ('べ', '連用形'), ('ます', '助動詞')],
+            '食べた': [('食', '動詞'), ('べ', '連用形'), ('た', '助動詞')],
+            '食べない': [('食', '動詞'), ('べ', '未然形'), ('ない', '助動詞')],
+            # 見る
+            '見ます': [('見', '動詞'), ('ます', '助動詞')],
+            '見た': [('見', '動詞'), ('た', '助動詞')],
+            '見ない': [('見', '動詞'), ('ない', '助動詞')],
+            # 買う
+            '買います': [('買', '動詞'), ('い', '連用形'), ('ます', '助動詞')],
+            '買った': [('買', '動詞'), ('っ', '促音便'), ('た', '助動詞')],
+            '買う': [('買', '動詞'), ('う', '終止形')],
+        }
+
+        # =================================================================
+        # 副詞 (부사)
+        # =================================================================
+        self.adverbs = {
+            'とても': '副詞', 'すごく': '副詞', 'かなり': '副詞',
+            'よく': '副詞', 'まだ': '副詞', 'もう': '副詞',
+            'たくさん': '副詞', 'ちょっと': '副詞', '少し': '副詞',
+        }
+
+        # =================================================================
+        # 확장 사전 로드 (optional external asset)
+        # =================================================================
+        self._load_extended_dictionary()
+
+    def _load_extended_dictionary(self):
+        """Load optional external extended dictionary"""
+        dict_path = DICT_DIR / 'ja_extended.json'
+        if not dict_path.exists():
+            return
+
+        with open(dict_path, 'r', encoding='utf-8') as f:
+            extended = json.load(f)
+
+        # 기존 사전에 추가
+        for word, upos in extended.items():
+            if upos in ('NOUN', 'PROPN'):
+                if word not in self.nouns:
+                    self.nouns[word] = ('名詞', word)
+            elif upos == 'VERB':
+                if word not in self.verbs:
+                    self.verbs[word] = ('動詞', word, 'godan')
+            elif upos == 'ADJ':
+                if word not in self.adjectives:
+                    self.adjectives[word] = ('形容詞', word)
+            elif upos == 'ADV':
+                if word not in self.adverbs:
+                    self.adverbs[word] = '副詞'
+
+    def _build_domain_dictionaries(self):
+        """도메인별 사전 구축"""
+
+        # FOOD 도메인
+        self._domain_dictionaries[Domain.FOOD] = {
+            '橋': ('箸', '名詞'),  # はし -> 젓가락
+            '株': ('株', '名詞'),  # かぶ -> 순무
+            '飯': ('飯', '名詞'),
+            '酒': ('酒', '名詞'),
+        }
+
+        # TECH 도메인
+        self._domain_dictionaries[Domain.TECH] = {
+            '橋': ('ブリッジ', '名詞'),  # 네트워크 브리지
+            '株': ('株', '名詞'),
+        }
+
+        # FINANCE 도메인
+        self._domain_dictionaries[Domain.FINANCE] = {
+            '橋': ('橋', '名詞'),  # 일반적 다리
+            '株': ('株式', '名詞'),  # 주식
+            '銀行': ('銀行', '名詞'),
+            '投資': ('投資', '名詞'),
+        }
+
+        # ENTERTAINMENT 도메인
+        self._domain_dictionaries[Domain.ENTERTAINMENT] = {
+            'AKB': ('AKB48', '名詞'),
+            '嵐': ('嵐', '名詞'),  # 아이돌 그룹
+        }
+
+    def _generate_candidates(self, text: str, domain: Domain) -> List[AnalysisResult]:
+        """분석 후보 생성"""
+        if not text or not text.strip():
+            return [AnalysisResult([])]
+
+        candidates = []
+
+        # 기본 분석
+        main_morphemes = self._analyze_text(text, domain)
+        main_result = AnalysisResult(
+            morphemes=main_morphemes,
+            score=1.0,
+            domain=domain
+        )
+        main_result.score = self._score_analysis(main_result)
+        candidates.append(main_result)
+
+        return candidates
+
+    def _analyze_text(self, text: str, domain: Domain) -> List[Morpheme]:
+        """텍스트 분석"""
+        if not text:
+            return []
+
+        result = []
+        pos = 0
+
+        while pos < len(text):
+            matched = False
+
+            # 공백 스킵
+            if text[pos].isspace():
+                pos += 1
+                continue
+
+            # 런타임 사전 우선 확인 (최장일치)
+            for length in range(min(len(text) - pos, 20), 0, -1):
+                substring = text[pos:pos+length]
+
+                if substring in self._user_dictionary:
+                    lemma, pos_tag, _ = self._user_dictionary[substring]
+                    result.append(Morpheme(
+                        surface=substring, lemma=lemma, pos=pos_tag,
+                        start=pos, end=pos + length
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+            if matched:
+                continue
+
+            # 도메인 사전 확인
+            for length in range(min(len(text) - pos, 10), 0, -1):
+                substring = text[pos:pos+length]
+                domain_sense = self._get_domain_sense(substring, domain)
+
+                if domain_sense:
+                    result.append(Morpheme(
+                        surface=substring, lemma=domain_sense[0], pos=domain_sense[1],
+                        start=pos, end=pos + length
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+            if matched:
+                continue
+
+            # 활용형 사전 확인
+            for length in range(min(len(text) - pos, 10), 0, -1):
+                substring = text[pos:pos+length]
+
+                if substring in self.verb_forms:
+                    curr_pos = pos
+                    for surface, tag in self.verb_forms[substring]:
+                        result.append(Morpheme(
+                            surface=surface, lemma=surface, pos=tag,
+                            start=curr_pos, end=curr_pos + len(surface)
+                        ))
+                        curr_pos += len(surface)
+                    pos += length
+                    matched = True
+                    break
+
+            if matched:
+                continue
+
+            # 최장일치 사전 탐색
+            for length in range(min(len(text) - pos, 10), 0, -1):
+                substring = text[pos:pos+length]
+
+                # 명사
+                if substring in self.nouns:
+                    info = self.nouns[substring]
+                    result.append(Morpheme(
+                        surface=substring, lemma=substring, pos=info[0],
+                        start=pos, end=pos + length,
+                        features={'reading': info[1]}
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+                # 조사
+                if substring in self.particles:
+                    result.append(Morpheme(
+                        surface=substring, lemma=substring, pos='助詞',
+                        start=pos, end=pos + length
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+                # 조동사
+                if substring in self.auxiliaries:
+                    result.append(Morpheme(
+                        surface=substring, lemma=substring, pos='助動詞',
+                        start=pos, end=pos + length
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+                # 부사
+                if substring in self.adverbs:
+                    result.append(Morpheme(
+                        surface=substring, lemma=substring, pos='副詞',
+                        start=pos, end=pos + length
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+                # 형용사
+                if substring in self.adjectives:
+                    info = self.adjectives[substring]
+                    result.append(Morpheme(
+                        surface=substring, lemma=info[1], pos=info[0],
+                        start=pos, end=pos + length
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+            if not matched:
+                # 스크립트별 청크 처리
+                char = text[pos]
+
+                # 한자
+                if self.KANJI_PATTERN.match(char):
+                    match = self.KANJI_PATTERN.match(text[pos:])
+                    chunk = match.group()
+                    result.append(Morpheme(
+                        surface=chunk, lemma=chunk, pos='名詞',
+                        start=pos, end=pos + len(chunk)
+                    ))
+                    pos += len(chunk)
+
+                # 히라가나
+                elif self.HIRAGANA_PATTERN.match(char):
+                    match = self.HIRAGANA_PATTERN.match(text[pos:])
+                    chunk = match.group()
+                    # 조사/조동사 분리 시도
+                    analyzed = self._analyze_hiragana_chunk(chunk, pos)
+                    result.extend(analyzed)
+                    pos += len(chunk)
+
+                # 가타카나 (외래어)
+                elif self.KATAKANA_PATTERN.match(char):
+                    match = self.KATAKANA_PATTERN.match(text[pos:])
+                    chunk = match.group()
+                    result.append(Morpheme(
+                        surface=chunk, lemma=chunk, pos='名詞',
+                        start=pos, end=pos + len(chunk),
+                        features={'type': 'katakana'}
+                    ))
+                    pos += len(chunk)
+
+                # 라틴 문자 (영어)
+                elif self.LATIN_PATTERN.match(char):
+                    match = self.LATIN_PATTERN.match(text[pos:])
+                    chunk = match.group()
+                    result.append(Morpheme(
+                        surface=chunk, lemma=chunk, pos='名詞',
+                        start=pos, end=pos + len(chunk),
+                        features={'type': 'latin'}
+                    ))
+                    pos += len(chunk)
+
+                # 숫자
+                elif self.NUMBER_PATTERN.match(char):
+                    match = self.NUMBER_PATTERN.match(text[pos:])
+                    chunk = match.group()
+                    result.append(Morpheme(
+                        surface=chunk, lemma=chunk, pos='数詞',
+                        start=pos, end=pos + len(chunk)
+                    ))
+                    pos += len(chunk)
+
+                # 기타 (기호 등)
+                else:
+                    result.append(Morpheme(
+                        surface=char, lemma=char, pos='記号',
+                        start=pos, end=pos + 1
+                    ))
+                    pos += 1
+
+        return result
+
+    def _analyze_hiragana_chunk(self, chunk: str, offset: int) -> List[Morpheme]:
+        """히라가나 청크 분석"""
+        results = []
+
+        # 조사/조동사 최장일치
+        pos = 0
+        while pos < len(chunk):
+            matched = False
+
+            for length in range(min(len(chunk) - pos, 5), 0, -1):
+                substring = chunk[pos:pos+length]
+
+                if substring in self.particles:
+                    results.append(Morpheme(
+                        surface=substring, lemma=substring, pos='助詞',
+                        start=offset + pos, end=offset + pos + length
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+                if substring in self.auxiliaries:
+                    results.append(Morpheme(
+                        surface=substring, lemma=substring, pos='助動詞',
+                        start=offset + pos, end=offset + pos + length
+                    ))
+                    pos += length
+                    matched = True
+                    break
+
+            if not matched:
+                # 남은 부분은 명사로
+                remaining = chunk[pos:]
+                if remaining:
+                    results.append(Morpheme(
+                        surface=remaining, lemma=remaining, pos='名詞',
+                        start=offset + pos, end=offset + len(chunk)
+                    ))
+                break
+
+        return results
+
+    def _generate_alternatives(self, text: str, domain: Domain, count: int) -> List[AnalysisResult]:
+        """대안 분석 결과 생성 (N-best용)"""
+        alternatives = []
+
+        # 다른 도메인으로 분석
+        other_domains = [d for d in Domain if d != domain][:count]
+
+        for alt_domain in other_domains:
+            morphemes = self._analyze_text(text, alt_domain)
+            result = AnalysisResult(
+                morphemes=morphemes,
+                score=0.8,
+                domain=alt_domain
+            )
+            result.score = self._score_analysis(result) * 0.9
+            alternatives.append(result)
+
+        return alternatives
+
+
+# Alias for backward compatibility
+JapaneseAnalyzer = JapaneseAdvancedAnalyzer